Evaluate YCSB Result¶

No description has been provided for this image

References:

  1. https://github.com/brianfrankcooper/YCSB/wiki/Running-a-Workload

Import Packages¶

In [1]:
#!pip list

%matplotlib inline

import pandas as pd
import os
import re
import matplotlib.pyplot as plt
import pickle
# Some nice output
pd.set_option("display.max_rows", None)
pd.set_option('display.max_colwidth', None)
from IPython.display import display, Markdown

import dbmsbenchmarker

import evaluator

Pick Result¶

In [2]:
code = "1684758882"
path = "./"
name_pattern = "PostgreSQL-32-(.+?)-(.+?)-1"
peak_benchmark = 65536
peak_loading = 2**14*2

Helper Functions¶

In [3]:
def plot_comparison(df_aggregated, x, y, colum, title, marker=None):
    if y in df_aggregated.columns:
        ax = evaluation.plot(df_aggregated.rename(columns={'pod_count': 'pods'}), column=column, x=x, y=y, figsize=(2,2))
        ax.title.set_size(10)
        ax.set_title(title)
        plt.legend(loc='lower right', title='num pods')
        if marker is not None:
            ax.axvline(marker, color='k', linestyle='--')
    else:
        print("{} not avalable".format(y))

def plot_variation(df_groups, max_pod_count, column):
    if column in df_groups:
        ax = df_groups[df_groups['pod_count']==max_pod_count].boxplot(by='target', column=column, rot=90, figsize=(2,2), grid=False)
        ax.get_figure().suptitle('')
        ax.title.set_size(10)
    else:
        print("{} not avalable".format(column))

def plot_metric(df, title, marker=None):
    global name_pattern
    df = pd.DataFrame(df, columns=["new"])
    #found = re.findall('PostgreSQL-AWS-32-(.+?)-(.+?)-1\n', stdout)
    #print(found)
    #df['vusers'] = 
    #print(name_pattern)
    #print(df)
    liste = list(df.index.map(lambda x: re.findall(name_pattern, x)))
    #print(liste)
    df['pods'] = [int(x[0][0]) for x in liste]
    df['target'] = [int(int(x[0][1])) for x in liste]
    #df['target'] = [int(int(x[0][1])/16384) for x in liste]
    #df['config'] = [x[0] for x in liste]
    #df.set_index('vusers', inplace=True)
    #ax = df.plot.bar(legend=False, figsize=(2,2), title="SUT CPU")
    #df.plot.bar(x='target')
    df = df.pivot(columns='pods', index='target', values='new')
    ax = df.plot(figsize=(2,2), title=title)
    plt.legend(title='num pods')
    ax.set_ylim(0, df.max().max()*1.05)
    ax.title.set_size(10)
    if marker is not None:
        ax.axvline(marker, color='k', linestyle='--')

Start Evaluation¶

In [4]:
evaluation = evaluator.ycsb(code=code, path=path)

Prepare Result¶

This has to be done only once. Prepared results will be stored in result folder.

Transform all Benchmarking Log Files to DataFrames¶

We also pick the first log file to be an example for later

In [5]:
filename_example = ""

directory = os.fsencode(path+"/"+code)
for file in os.listdir(directory):
    filename = os.fsdecode(file)
    if filename.startswith("bexhoma-benchmarker") and filename.endswith(".log"):
        #print("filename:", filename)
        pod_name = filename[filename.rindex("-")+1:-len(".log")]
        #print("pod_name:", pod_name)
        jobname = filename[len("bexhoma-benchmarker-"):-len("-"+pod_name+".log")]
        #print("jobname:", jobname)
        evaluation.end_benchmarking(jobname)
        filename_example = filename

Show a DataFrame for Single Pod as Example¶

In [6]:
filename = path+"/"+code+"/"+filename_example+".df.pickle"
print(filename)
df = pd.read_pickle(filename)
df

.//1684758882/bexhoma-benchmarker-postgresql-32-8-98304-1684758882-1-1-wjjww.log.df.pickle
Out[6]:
connection configuration experiment_run client pod pod_count threads target sf workload ... [CLEANUP].MaxLatency(us) [CLEANUP].95thPercentileLatency(us) [CLEANUP].99thPercentileLatency(us) [UPDATE].Operations [UPDATE].AverageLatency(us) [UPDATE].MinLatency(us) [UPDATE].MaxLatency(us) [UPDATE].95thPercentileLatency(us) [UPDATE].99thPercentileLatency(us) [UPDATE].Return=OK
PostgreSQL-32-8-98304-1
0 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304 1 1 wjjww 8 4 12288 30 a ... 299 299 299 1874342 640.6968872276244 104 18579455 1043 3525 1874342

1 rows × 43 columns

Transform all Loading Log Files to DataFrames¶

We also pick the first log file to be an example for later

In [7]:
directory = os.fsencode(path+"/"+code)
for file in os.listdir(directory):
    filename = os.fsdecode(file)
    if filename.startswith("bexhoma-loading") and filename.endswith(".sensor.log"):
        #print("filename:", filename)
        pod_name = filename[filename.rindex("-")+1:-len(".log")]
        #print("pod_name:", pod_name)
        jobname = filename[len("bexhoma-loading-"):-len("-"+pod_name+".sensor.log")]
        #print("jobname:", jobname)
        evaluation.end_loading(jobname)

Show a DataFrame for Single Pod as Example¶

In [8]:
filename = path+"/"+code+"/"+filename_example+".df.pickle"
print(filename)
df = pd.read_pickle(filename)
df

.//1684758882/bexhoma-benchmarker-postgresql-32-8-98304-1684758882-1-1-wjjww.log.df.pickle
Out[8]:
connection configuration experiment_run client pod pod_count threads target sf workload ... [CLEANUP].MaxLatency(us) [CLEANUP].95thPercentileLatency(us) [CLEANUP].99thPercentileLatency(us) [UPDATE].Operations [UPDATE].AverageLatency(us) [UPDATE].MinLatency(us) [UPDATE].MaxLatency(us) [UPDATE].95thPercentileLatency(us) [UPDATE].99thPercentileLatency(us) [UPDATE].Return=OK
PostgreSQL-32-8-98304-1
0 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304 1 1 wjjww 8 4 12288 30 a ... 299 299 299 1874342 640.6968872276244 104 18579455 1043 3525 1874342

1 rows × 43 columns

Transform all DataFrames into Single Result DataFrame¶

In [9]:
evaluation.evaluate_results()

Get Loading Result¶

In [10]:
df = evaluation.get_df_loading()

df = df.sort_values(['configuration','experiment_run','client'])

df = df[df.columns.drop(list(df.filter(regex='FAILED')))]

df.T
Out[10]:
connection_pod PostgreSQL-32-1-114688-1 PostgreSQL-32-1-131072-1 PostgreSQL-32-1-16384-1 PostgreSQL-32-1-32768-1 PostgreSQL-32-1-49152-1 PostgreSQL-32-1-65536-1 PostgreSQL-32-1-81920-1 PostgreSQL-32-1-98304-1 PostgreSQL-32-8-114688-1 PostgreSQL-32-8-114688-2 ... PostgreSQL-32-8-81920-7 PostgreSQL-32-8-81920-8 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304-2 PostgreSQL-32-8-98304-3 PostgreSQL-32-8-98304-4 PostgreSQL-32-8-98304-5 PostgreSQL-32-8-98304-6 PostgreSQL-32-8-98304-7 PostgreSQL-32-8-98304-8
connection PostgreSQL-32-1-114688 PostgreSQL-32-1-131072 PostgreSQL-32-1-16384 PostgreSQL-32-1-32768 PostgreSQL-32-1-49152 PostgreSQL-32-1-65536 PostgreSQL-32-1-81920 PostgreSQL-32-1-98304 PostgreSQL-32-8-114688 PostgreSQL-32-8-114688 ... PostgreSQL-32-8-81920 PostgreSQL-32-8-81920 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304
configuration PostgreSQL-32-1-114688 PostgreSQL-32-1-131072 PostgreSQL-32-1-16384 PostgreSQL-32-1-32768 PostgreSQL-32-1-49152 PostgreSQL-32-1-65536 PostgreSQL-32-1-81920 PostgreSQL-32-1-98304 PostgreSQL-32-8-114688 PostgreSQL-32-8-114688 ... PostgreSQL-32-8-81920 PostgreSQL-32-8-81920 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304
experiment_run 1 1 1 1 1 1 1 1 1 1 ... 1 1 1 1 1 1 1 1 1 1
client 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
pod tx5d5.sensor b9gzf.sensor mr62r.sensor 9fbk6.sensor gs5fb.sensor 6hqzb.sensor nkd9n.sensor xzd7w.sensor 5x62v.sensor 7bc2c.sensor ... qpc6h.sensor vtqn4.sensor 6xx8q.sensor ghmf9.sensor gqw9m.sensor jln48.sensor n9xct.sensor p78x4.sensor s4t2s.sensor zklhx.sensor
pod_count 1 1 1 1 1 1 1 1 8 8 ... 8 8 8 8 8 8 8 8 8 8
threads 32 32 32 32 32 32 32 32 4 4 ... 4 4 4 4 4 4 4 4 4 4
target 114688 131072 16384 32768 49152 65536 81920 98304 14336 14336 ... 10240 10240 12288 12288 12288 12288 12288 12288 12288 12288
sf 30 30 30 30 30 30 30 30 30 30 ... 30 30 30 30 30 30 30 30 30 30
workload a a a a a a a a a a ... a a a a a a a a a a
operations 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 3750000 3750000 ... 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000
batchsize -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 ... -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
[OVERALL].RunTime(ms) 1066169 1085442 1831226 1059352 1062779 1030604 1054230 1074369 1032151 1039143 ... 1004788 1073472 1054586 1069092 1054273 979125 1054125 1013255 970744 1036959
[OVERALL].Throughput(ops/sec) 28138.128195436184 27638.51039484376 16382.467265099993 28319.198906501333 28227.88180797701 29109.143764239223 28456.788366864916 27923.367111299747 3633.189329855806 3608.74297377743 ... 3732.1305588840632 3493.337506707208 3555.8977646204294 3507.6494819903246 3556.9534646149527 3829.9502106472614 3557.4528637495555 3700.943987446398 3863.0164080334257 3616.3435584242
[TOTAL_GCS_Copy].Count 1662 1662 1662 1662 1662 1662 1662 1662 207 207 ... 207 207 207 207 207 207 207 207 207 207
[TOTAL_GC_TIME_Copy].Time(ms) 1411 1425 1361 1404 1412 1393 1403 1409 160 161 ... 160 161 161 160 161 159 159 161 165 161
[TOTAL_GC_TIME_%_Copy].Time(%) 0.1323429962792015 0.13128292437550787 0.07432179315933697 0.13253385088242622 0.1328592303762118 0.1351634575452841 0.13308291359570493 0.1311467475327378 0.01550160780738477 0.015493536500751099 ... 0.01592375705123867 0.014998062362129612 0.015266654402770375 0.014965971123158717 0.015271186874746864 0.01623898889314439 0.015083600142298115 0.0158893861861032 0.016997272195347076 0.0155261683441679
[TOTAL_GCS_MarkSweepCompact].Count 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
[TOTAL_GC_TIME_MarkSweepCompact].Time(ms) 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
[TOTAL_GC_TIME_%_MarkSweepCompact].Time(%) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
[TOTAL_GCs].Count 1662 1662 1662 1662 1662 1662 1662 1662 207 207 ... 207 207 207 207 207 207 207 207 207 207
[TOTAL_GC_TIME].Time(ms) 1411 1425 1361 1404 1412 1393 1403 1409 160 161 ... 160 161 161 160 161 159 159 161 165 161
[TOTAL_GC_TIME_%].Time(%) 0.1323429962792015 0.13128292437550787 0.07432179315933697 0.13253385088242622 0.1328592303762118 0.1351634575452841 0.13308291359570493 0.1311467475327378 0.01550160780738477 0.015493536500751099 ... 0.01592375705123867 0.014998062362129612 0.015266654402770375 0.014965971123158717 0.015271186874746864 0.01623898889314439 0.015083600142298115 0.0158893861861032 0.016997272195347076 0.0155261683441679
[CLEANUP].Operations 32 32 32 32 32 32 32 32 4 4 ... 4 4 4 4 4 4 4 4 4 4
[CLEANUP].AverageLatency(us) 95.4375 95.0 140.40625 99.6875 90.25 86.625 95.25 95.5 116.25 162.0 ... 123.75 133.75 128.5 118.25 109.25 139.5 142.5 153.0 104.75 125.5
[CLEANUP].MinLatency(us) 57 37 32 53 47 55 54 54 61 72 ... 61 76 76 62 63 104 103 105 59 62
[CLEANUP].MaxLatency(us) 186 199 692 195 199 206 214 375 187 373 ... 196 217 192 200 195 212 237 287 234 260
[CLEANUP].95thPercentileLatency(us) 144 154 450 140 147 132 129 115 187 373 ... 196 217 192 200 195 212 237 287 234 260
[CLEANUP].99thPercentileLatency(us) 186 199 692 195 199 206 214 375 187 373 ... 196 217 192 200 195 212 237 287 234 260
[INSERT].Operations 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 3750000 3750000 ... 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000
[INSERT].AverageLatency(us) 1024.3594275 1033.9956250666667 835.0811325 1019.0929389666667 1042.6627607666667 1032.8564868 1039.4797652333334 1031.2387599666667 1071.3269501333334 1064.0484906666666 ... 1021.6099389333333 1072.3814525333332 1045.2016928 1028.8221322666666 1045.3850088 1015.8059450666667 1083.8865906666667 1051.2896445333333 991.6972021333333 1022.2611192
[INSERT].MinLatency(us) 629 636 633 630 630 629 636 630 653 687 ... 645 686 632 631 639 636 689 638 638 636
[INSERT].MaxLatency(us) 18612223 24150015 385791 26443775 23822335 23887871 23085055 23478271 23658495 23658495 ... 25804799 25804799 23363583 23363583 23363583 23363583 23363583 23363583 23363583 23363583
[INSERT].95thPercentileLatency(us) 967 945 1025 1018 981 967 967 959 1034 1015 ... 943 1009 1000 998 1011 919 1001 987 914 955
[INSERT].99thPercentileLatency(us) 1261 1268 1180 1261 1268 1255 1261 1263 1278 1247 ... 1173 1306 1229 1251 1238 1164 1246 1220 1162 1198
[INSERT].Return=OK 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 3750000 3750000 ... 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000

36 rows × 72 columns

Plot Results¶

Set Data Types for Plotting¶

In [11]:
df.fillna(0, inplace=True)

df_plot = evaluation.loading_set_datatypes(df)

df_plot.sort_values('target', inplace=True)

Remove Failed Pods¶

In [12]:
df_plot.drop(df_plot[df_plot['[OVERALL].Throughput(ops/sec)'] == 0].index, inplace=True)

df_plot.T
Out[12]:
connection_pod PostgreSQL-32-8-16384-2 PostgreSQL-32-8-16384-3 PostgreSQL-32-8-16384-4 PostgreSQL-32-8-16384-5 PostgreSQL-32-8-16384-6 PostgreSQL-32-8-16384-7 PostgreSQL-32-8-16384-8 PostgreSQL-32-8-16384-1 PostgreSQL-32-8-32768-4 PostgreSQL-32-8-32768-1 ... PostgreSQL-32-8-131072-6 PostgreSQL-32-8-131072-7 PostgreSQL-32-8-131072-1 PostgreSQL-32-1-32768-1 PostgreSQL-32-1-49152-1 PostgreSQL-32-1-65536-1 PostgreSQL-32-1-81920-1 PostgreSQL-32-1-98304-1 PostgreSQL-32-1-114688-1 PostgreSQL-32-1-131072-1
connection PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-32768 PostgreSQL-32-8-32768 ... PostgreSQL-32-8-131072 PostgreSQL-32-8-131072 PostgreSQL-32-8-131072 PostgreSQL-32-1-32768 PostgreSQL-32-1-49152 PostgreSQL-32-1-65536 PostgreSQL-32-1-81920 PostgreSQL-32-1-98304 PostgreSQL-32-1-114688 PostgreSQL-32-1-131072
configuration PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 PostgreSQL-32-8-32768 PostgreSQL-32-8-32768 ... PostgreSQL-32-8-131072 PostgreSQL-32-8-131072 PostgreSQL-32-8-131072 PostgreSQL-32-1-32768 PostgreSQL-32-1-49152 PostgreSQL-32-1-65536 PostgreSQL-32-1-81920 PostgreSQL-32-1-98304 PostgreSQL-32-1-114688 PostgreSQL-32-1-131072
experiment_run 1 1 1 1 1 1 1 1 1 1 ... 1 1 1 1 1 1 1 1 1 1
client 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
pod gb9j5.sensor kjk85.sensor mskcx.sensor pcmmg.sensor pmr6w.sensor qmfjw.sensor xrrbk.sensor 2hl8g.sensor lvlv2.sensor 5m288.sensor ... p6x48.sensor p9pwx.sensor 2dbvt.sensor 9fbk6.sensor gs5fb.sensor 6hqzb.sensor nkd9n.sensor xzd7w.sensor tx5d5.sensor b9gzf.sensor
pod_count 8 8 8 8 8 8 8 8 8 8 ... 8 8 8 1 1 1 1 1 1 1
threads 4 4 4 4 4 4 4 4 4 4 ... 4 4 4 32 32 32 32 32 32 32
target 2048 2048 2048 2048 2048 2048 2048 2048 4096 4096 ... 16384 16384 16384 32768 49152 65536 81920 98304 114688 131072
sf 30 30 30 30 30 30 30 30 30 30 ... 30 30 30 30 30 30 30 30 30 30
workload a a a a a a a a a a ... a a a a a a a a a a
operations 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 ... 3750000 3750000 3750000 30000000 30000000 30000000 30000000 30000000 30000000 30000000
batchsize -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 ... -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
[OVERALL].RunTime(ms) 1831208.0 1831207.0 1831193.0 1831201.0 1831203.0 1831193.0 1831197.0 1831205.0 1046118.0 1070644.0 ... 1044507.0 1016913.0 1019260.0 1059352.0 1062779.0 1030604.0 1054230.0 1074369.0 1066169.0 1085442.0
[OVERALL].Throughput(ops/sec) 2047.828537 2047.829656 2047.845312 2047.836365 2047.834129 2047.845312 2047.840839 2047.831892 3584.681652 3502.564811 ... 3590.210501 3687.631095 3679.139768 28319.198907 28227.881808 29109.143764 28456.788367 27923.367111 28138.128195 27638.510395
[TOTAL_GCS_Copy].Count 207 207 207 207 207 207 207 207 207 206 ... 207 207 207 1662 1662 1662 1662 1662 1662 1662
[TOTAL_GC_TIME_Copy].Time(ms) 152 158 157 153 156 156 156 157 163 166 ... 161 161 159 1404 1412 1393 1403 1409 1411 1425
[TOTAL_GC_TIME_%_Copy].Time(%) 0.008300531670897025 0.008628188948600568 0.008573645705286117 0.008355172370482541 0.008518989975442374 0.008519036496972193 0.008519017888299294 0.008573589521653775 0.015581416245586061 0.015504686898726373 ... 0.01541397041858025 0.015832229502425476 0.015599552616604203 0.13253385088242622 0.1328592303762118 0.1351634575452841 0.13308291359570493 0.1311467475327378 0.1323429962792015 0.13128292437550787
[TOTAL_GCS_MarkSweepCompact].Count 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
[TOTAL_GC_TIME_MarkSweepCompact].Time(ms) 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
[TOTAL_GC_TIME_%_MarkSweepCompact].Time(%) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
[TOTAL_GCs].Count 207 207 207 207 207 207 207 207 207 206 ... 207 207 207 1662 1662 1662 1662 1662 1662 1662
[TOTAL_GC_TIME].Time(ms) 152 158 157 153 156 156 156 157 163 166 ... 161 161 159 1404 1412 1393 1403 1409 1411 1425
[TOTAL_GC_TIME_%].Time(%) 0.008300531670897025 0.008628188948600568 0.008573645705286117 0.008355172370482541 0.008518989975442374 0.008519036496972193 0.008519017888299294 0.008573589521653775 0.015581416245586061 0.015504686898726373 ... 0.01541397041858025 0.015832229502425476 0.015599552616604203 0.13253385088242622 0.1328592303762118 0.1351634575452841 0.13308291359570493 0.1311467475327378 0.1323429962792015 0.13128292437550787
[CLEANUP].Operations 4 4 4 4 4 4 4 4 4 4 ... 4 4 4 32 32 32 32 32 32 32
[CLEANUP].AverageLatency(us) 115.25 79.25 84.25 85.75 82.25 182.25 83.0 224.75 220.25 205.5 ... 139.25 116.5 104.75 99.6875 90.25 86.625 95.25 95.5 95.4375 95.0
[CLEANUP].MinLatency(us) 44.0 33.0 49.0 45.0 39.0 113.0 43.0 49.0 115.0 65.0 ... 62.0 63.0 60.0 53.0 47.0 55.0 54.0 54.0 57.0 37.0
[CLEANUP].MaxLatency(us) 214.0 187.0 188.0 191.0 186.0 251.0 186.0 423.0 470.0 506.0 ... 207.0 211.0 214.0 195.0 199.0 206.0 214.0 375.0 186.0 199.0
[CLEANUP].95thPercentileLatency(us) 214.0 187.0 188.0 191.0 186.0 251.0 186.0 423.0 470.0 506.0 ... 207.0 211.0 214.0 140.0 147.0 132.0 129.0 115.0 144.0 154.0
[CLEANUP].99thPercentileLatency(us) 214.0 187.0 188.0 191.0 186.0 251.0 186.0 423.0 470.0 506.0 ... 207.0 211.0 214.0 195.0 199.0 206.0 214.0 375.0 186.0 199.0
[INSERT].Operations 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 ... 3750000 3750000 3750000 30000000 30000000 30000000 30000000 30000000 30000000 30000000
[INSERT].AverageLatency(us) 832.772679 812.780115 839.590646 790.366524 856.366943 893.798743 810.112466 855.068822 1008.154887 1017.997042 ... 1026.584293 989.811878 1018.979515 1019.092939 1042.662761 1032.856487 1039.479765 1031.23876 1024.359428 1033.995625
[INSERT].MinLatency(us) 647.0 646.0 637.0 646.0 643.0 650.0 649.0 643.0 634.0 635.0 ... 637.0 640.0 640.0 630.0 630.0 629.0 636.0 630.0 629.0 636.0
[INSERT].MaxLatency(us) 142975.0 142975.0 143103.0 142975.0 143231.0 143231.0 143103.0 143103.0 25477119.0 25477119.0 ... 22003711.0 22003711.0 22003711.0 26443775.0 23822335.0 23887871.0 23085055.0 23478271.0 18612223.0 24150015.0
[INSERT].95thPercentileLatency(us) 1025.0 977.0 1034.0 973.0 1107.0 1134.0 1000.0 1063.0 984.0 1027.0 ... 1017.0 939.0 961.0 1018.0 981.0 967.0 967.0 959.0 967.0 945.0
[INSERT].99thPercentileLatency(us) 1129.0 1089.0 1220.0 1044.0 1275.0 1352.0 1213.0 1201.0 1248.0 1298.0 ... 1254.0 1149.0 1203.0 1261.0 1268.0 1255.0 1261.0 1263.0 1261.0 1268.0
[INSERT].Return=OK 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 ... 3750000 3750000 3750000 30000000 30000000 30000000 30000000 30000000 30000000 30000000

36 rows × 72 columns

Plot Results per Number of Pods¶

In [13]:
%matplotlib inline

column = "threads"
x = "target"
y = "[OVERALL].Throughput(ops/sec)"
evaluation.plot(df_plot, column=column, x=x, y=y, plot_by="pod_count")
No description has been provided for this image
In [14]:
column = ["threads", "pod_count"]
x = "target"
y = "[INSERT].95thPercentileLatency(us)"
evaluation.plot(df_plot, column=column, x=x, y=y, plot_by="experiment_run")

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
No description has been provided for this image

Aggregate by Parallel Pods¶

In [15]:
df = evaluation.get_df_loading()
df
Out[15]:
connection configuration experiment_run client pod pod_count threads target sf workload ... [CLEANUP].MaxLatency(us) [CLEANUP].95thPercentileLatency(us) [CLEANUP].99thPercentileLatency(us) [INSERT].Operations [INSERT].AverageLatency(us) [INSERT].MinLatency(us) [INSERT].MaxLatency(us) [INSERT].95thPercentileLatency(us) [INSERT].99thPercentileLatency(us) [INSERT].Return=OK
connection_pod
PostgreSQL-32-1-114688-1 PostgreSQL-32-1-114688 PostgreSQL-32-1-114688 1 0 tx5d5.sensor 1 32 114688 30 a ... 186 144 186 30000000 1024.3594275 629 18612223 967 1261 30000000
PostgreSQL-32-1-131072-1 PostgreSQL-32-1-131072 PostgreSQL-32-1-131072 1 0 b9gzf.sensor 1 32 131072 30 a ... 199 154 199 30000000 1033.9956250666667 636 24150015 945 1268 30000000
PostgreSQL-32-1-16384-1 PostgreSQL-32-1-16384 PostgreSQL-32-1-16384 1 0 mr62r.sensor 1 32 16384 30 a ... 692 450 692 30000000 835.0811325 633 385791 1025 1180 30000000
PostgreSQL-32-1-32768-1 PostgreSQL-32-1-32768 PostgreSQL-32-1-32768 1 0 9fbk6.sensor 1 32 32768 30 a ... 195 140 195 30000000 1019.0929389666667 630 26443775 1018 1261 30000000
PostgreSQL-32-1-49152-1 PostgreSQL-32-1-49152 PostgreSQL-32-1-49152 1 0 gs5fb.sensor 1 32 49152 30 a ... 199 147 199 30000000 1042.6627607666667 630 23822335 981 1268 30000000
PostgreSQL-32-1-65536-1 PostgreSQL-32-1-65536 PostgreSQL-32-1-65536 1 0 6hqzb.sensor 1 32 65536 30 a ... 206 132 206 30000000 1032.8564868 629 23887871 967 1255 30000000
PostgreSQL-32-1-81920-1 PostgreSQL-32-1-81920 PostgreSQL-32-1-81920 1 0 nkd9n.sensor 1 32 81920 30 a ... 214 129 214 30000000 1039.4797652333334 636 23085055 967 1261 30000000
PostgreSQL-32-1-98304-1 PostgreSQL-32-1-98304 PostgreSQL-32-1-98304 1 0 xzd7w.sensor 1 32 98304 30 a ... 375 115 375 30000000 1031.2387599666667 630 23478271 959 1263 30000000
PostgreSQL-32-8-114688-1 PostgreSQL-32-8-114688 PostgreSQL-32-8-114688 1 0 5x62v.sensor 8 4 14336 30 a ... 187 187 187 3750000 1071.3269501333334 653 23658495 1034 1278 3750000
PostgreSQL-32-8-114688-2 PostgreSQL-32-8-114688 PostgreSQL-32-8-114688 1 0 7bc2c.sensor 8 4 14336 30 a ... 373 373 373 3750000 1064.0484906666666 687 23658495 1015 1247 3750000
PostgreSQL-32-8-114688-3 PostgreSQL-32-8-114688 PostgreSQL-32-8-114688 1 0 c7w5h.sensor 8 4 14336 30 a ... 359 359 359 3750000 1062.6480872 698 23658495 1068 1321 3750000
PostgreSQL-32-8-114688-4 PostgreSQL-32-8-114688 PostgreSQL-32-8-114688 1 0 dg57m.sensor 8 4 14336 30 a ... 202 202 202 3750000 1054.2813354666666 639 23658495 988 1227 3750000
PostgreSQL-32-8-114688-5 PostgreSQL-32-8-114688 PostgreSQL-32-8-114688 1 0 dn5vx.sensor 8 4 14336 30 a ... 448 448 448 3750000 1046.1617666666666 634 23658495 1002 1243 3750000
PostgreSQL-32-8-114688-6 PostgreSQL-32-8-114688 PostgreSQL-32-8-114688 1 0 hkqrv.sensor 8 4 14336 30 a ... 200 200 200 3750000 1015.1118 643 23658495 915 1154 3750000
PostgreSQL-32-8-114688-7 PostgreSQL-32-8-114688 PostgreSQL-32-8-114688 1 0 qbf55.sensor 8 4 14336 30 a ... 217 217 217 3750000 979.8143810666667 629 23658495 930 1164 3750000
PostgreSQL-32-8-114688-8 PostgreSQL-32-8-114688 PostgreSQL-32-8-114688 1 0 sr6t8.sensor 8 4 14336 30 a ... 195 195 195 3750000 1062.6591456 682 23658495 1034 1260 3750000
PostgreSQL-32-8-131072-1 PostgreSQL-32-8-131072 PostgreSQL-32-8-131072 1 0 2dbvt.sensor 8 4 16384 30 a ... 214 214 214 3750000 1018.9795154666666 640 22003711 961 1203 3750000
PostgreSQL-32-8-131072-2 PostgreSQL-32-8-131072 PostgreSQL-32-8-131072 1 0 2vfvb.sensor 8 4 16384 30 a ... 193 193 193 3750000 1037.8515736 632 22003711 950 1194 3750000
PostgreSQL-32-8-131072-3 PostgreSQL-32-8-131072 PostgreSQL-32-8-131072 1 0 5qftv.sensor 8 4 16384 30 a ... 368 368 368 3750000 1108.4157389333334 642 22003711 1077 1333 3750000
PostgreSQL-32-8-131072-4 PostgreSQL-32-8-131072 PostgreSQL-32-8-131072 1 0 fwkgs.sensor 8 4 16384 30 a ... 336 336 336 3750000 1011.0584301333333 635 22003711 1018 1246 3750000
PostgreSQL-32-8-131072-5 PostgreSQL-32-8-131072 PostgreSQL-32-8-131072 1 0 n8gtm.sensor 8 4 16384 30 a ... 266 266 266 3750000 971.4101096 636 22003711 851 1042 3750000
PostgreSQL-32-8-131072-6 PostgreSQL-32-8-131072 PostgreSQL-32-8-131072 1 0 p6x48.sensor 8 4 16384 30 a ... 207 207 207 3750000 1026.5842928 637 22003711 1017 1254 3750000
PostgreSQL-32-8-131072-7 PostgreSQL-32-8-131072 PostgreSQL-32-8-131072 1 0 p9pwx.sensor 8 4 16384 30 a ... 211 211 211 3750000 989.8118778666667 640 22003711 939 1149 3750000
PostgreSQL-32-8-131072-8 PostgreSQL-32-8-131072 PostgreSQL-32-8-131072 1 0 vjc59.sensor 8 4 16384 30 a ... 279 279 279 3750000 1058.5769722666666 636 22003711 965 1194 3750000
PostgreSQL-32-8-16384-1 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 1 0 2hl8g.sensor 8 4 2048 30 a ... 423 423 423 3750000 855.0688224 643 143103 1063 1201 3750000
PostgreSQL-32-8-16384-2 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 1 0 gb9j5.sensor 8 4 2048 30 a ... 214 214 214 3750000 832.7726792 647 142975 1025 1129 3750000
PostgreSQL-32-8-16384-3 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 1 0 kjk85.sensor 8 4 2048 30 a ... 187 187 187 3750000 812.7801149333334 646 142975 977 1089 3750000
PostgreSQL-32-8-16384-4 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 1 0 mskcx.sensor 8 4 2048 30 a ... 188 188 188 3750000 839.5906464 637 143103 1034 1220 3750000
PostgreSQL-32-8-16384-5 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 1 0 pcmmg.sensor 8 4 2048 30 a ... 191 191 191 3750000 790.3665242666667 646 142975 973 1044 3750000
PostgreSQL-32-8-16384-6 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 1 0 pmr6w.sensor 8 4 2048 30 a ... 186 186 186 3750000 856.3669426666667 643 143231 1107 1275 3750000
PostgreSQL-32-8-16384-7 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 1 0 qmfjw.sensor 8 4 2048 30 a ... 251 251 251 3750000 893.7987432 650 143231 1134 1352 3750000
PostgreSQL-32-8-16384-8 PostgreSQL-32-8-16384 PostgreSQL-32-8-16384 1 0 xrrbk.sensor 8 4 2048 30 a ... 186 186 186 3750000 810.1124664 649 143103 1000 1213 3750000
PostgreSQL-32-8-32768-1 PostgreSQL-32-8-32768 PostgreSQL-32-8-32768 1 0 5m288.sensor 8 4 4096 30 a ... 506 506 506 3750000 1017.9970421333334 635 25477119 1027 1298 3750000
PostgreSQL-32-8-32768-2 PostgreSQL-32-8-32768 PostgreSQL-32-8-32768 1 0 dxlcv.sensor 8 4 4096 30 a ... 217 217 217 3750000 1018.7530533333334 642 25477119 991 1275 3750000
PostgreSQL-32-8-32768-3 PostgreSQL-32-8-32768 PostgreSQL-32-8-32768 1 0 j97qc.sensor 8 4 4096 30 a ... 190 190 190 3750000 1013.7664738666666 641 25477119 987 1317 3750000
PostgreSQL-32-8-32768-4 PostgreSQL-32-8-32768 PostgreSQL-32-8-32768 1 0 lvlv2.sensor 8 4 4096 30 a ... 470 470 470 3750000 1008.1548872 634 25477119 984 1248 3750000
PostgreSQL-32-8-32768-5 PostgreSQL-32-8-32768 PostgreSQL-32-8-32768 1 0 nzswp.sensor 8 4 4096 30 a ... 192 192 192 3750000 1050.2617424 641 25477119 1024 1342 3750000
PostgreSQL-32-8-32768-6 PostgreSQL-32-8-32768 PostgreSQL-32-8-32768 1 0 qk7pg.sensor 8 4 4096 30 a ... 204 204 204 3750000 982.0317464 635 25477119 959 1140 3750000
PostgreSQL-32-8-32768-7 PostgreSQL-32-8-32768 PostgreSQL-32-8-32768 1 0 thvmb.sensor 8 4 4096 30 a ... 194 194 194 3750000 1003.5151149333333 634 25477119 993 1229 3750000
PostgreSQL-32-8-32768-8 PostgreSQL-32-8-32768 PostgreSQL-32-8-32768 1 0 vbs9b.sensor 8 4 4096 30 a ... 474 474 474 3750000 1025.1049664 643 25477119 1007 1229 3750000
PostgreSQL-32-8-49152-1 PostgreSQL-32-8-49152 PostgreSQL-32-8-49152 1 0 7fnlv.sensor 8 4 6144 30 a ... 373 373 373 3750000 1041.5667581333332 636 23117823 938 1178 3750000
PostgreSQL-32-8-49152-2 PostgreSQL-32-8-49152 PostgreSQL-32-8-49152 1 0 8pcnm.sensor 8 4 6144 30 a ... 213 213 213 3750000 1032.7283445333333 640 23101439 889 1153 3750000
PostgreSQL-32-8-49152-3 PostgreSQL-32-8-49152 PostgreSQL-32-8-49152 1 0 9kkpj.sensor 8 4 6144 30 a ... 189 189 189 3750000 1058.0164234666668 640 23117823 946 1239 3750000
PostgreSQL-32-8-49152-4 PostgreSQL-32-8-49152 PostgreSQL-32-8-49152 1 0 jlvdb.sensor 8 4 6144 30 a ... 190 190 190 3750000 1018.2560928 636 23101439 912 1162 3750000
PostgreSQL-32-8-49152-5 PostgreSQL-32-8-49152 PostgreSQL-32-8-49152 1 0 m7bxp.sensor 8 4 6144 30 a ... 198 198 198 3750000 1020.9561941333334 633 23101439 862 1116 3750000
PostgreSQL-32-8-49152-6 PostgreSQL-32-8-49152 PostgreSQL-32-8-49152 1 0 s229l.sensor 8 4 6144 30 a ... 210 210 210 3750000 1035.5929082666667 639 23101439 930 1207 3750000
PostgreSQL-32-8-49152-7 PostgreSQL-32-8-49152 PostgreSQL-32-8-49152 1 0 sb97n.sensor 8 4 6144 30 a ... 222 222 222 3750000 1074.3827728 649 23101439 964 1237 3750000
PostgreSQL-32-8-49152-8 PostgreSQL-32-8-49152 PostgreSQL-32-8-49152 1 0 x2j79.sensor 8 4 6144 30 a ... 205 205 205 3750000 1011.5114624 630 23101439 859 1105 3750000
PostgreSQL-32-8-65536-1 PostgreSQL-32-8-65536 PostgreSQL-32-8-65536 1 0 42qlk.sensor 8 4 8192 30 a ... 210 210 210 3750000 996.8832234666667 630 22495231 990 1205 3750000
PostgreSQL-32-8-65536-2 PostgreSQL-32-8-65536 PostgreSQL-32-8-65536 1 0 5dqdh.sensor 8 4 8192 30 a ... 404 404 404 3750000 1075.0937845333333 702 22495231 1048 1269 3750000
PostgreSQL-32-8-65536-3 PostgreSQL-32-8-65536 PostgreSQL-32-8-65536 1 0 66dgh.sensor 8 4 8192 30 a ... 436 436 436 3750000 983.9162618666667 637 22495231 942 1171 3750000
PostgreSQL-32-8-65536-4 PostgreSQL-32-8-65536 PostgreSQL-32-8-65536 1 0 6778k.sensor 8 4 8192 30 a ... 373 373 373 3750000 1075.2453562666667 651 22495231 1070 1293 3750000
PostgreSQL-32-8-65536-5 PostgreSQL-32-8-65536 PostgreSQL-32-8-65536 1 0 6vllq.sensor 8 4 8192 30 a ... 354 354 354 3750000 1086.0009186666666 650 22495231 1105 1350 3750000
PostgreSQL-32-8-65536-6 PostgreSQL-32-8-65536 PostgreSQL-32-8-65536 1 0 d8z5q.sensor 8 4 8192 30 a ... 251 251 251 3750000 1005.459416 642 22495231 970 1191 3750000
PostgreSQL-32-8-65536-7 PostgreSQL-32-8-65536 PostgreSQL-32-8-65536 1 0 k8drx.sensor 8 4 8192 30 a ... 230 230 230 3750000 987.9829925333333 635 22495231 985 1216 3750000
PostgreSQL-32-8-65536-8 PostgreSQL-32-8-65536 PostgreSQL-32-8-65536 1 0 m4t2w.sensor 8 4 8192 30 a ... 210 210 210 3750000 1054.6162472 696 22495231 1030 1261 3750000
PostgreSQL-32-8-81920-1 PostgreSQL-32-8-81920 PostgreSQL-32-8-81920 1 0 2gmfh.sensor 8 4 10240 30 a ... 205 205 205 3750000 1040.1317130666666 630 25804799 943 1197 3750000
PostgreSQL-32-8-81920-2 PostgreSQL-32-8-81920 PostgreSQL-32-8-81920 1 0 7nlbl.sensor 8 4 10240 30 a ... 221 221 221 3750000 995.7930290666667 632 25804799 901 1142 3750000
PostgreSQL-32-8-81920-3 PostgreSQL-32-8-81920 PostgreSQL-32-8-81920 1 0 b2bbf.sensor 8 4 10240 30 a ... 185 185 185 3750000 1064.0971224 686 25804799 996 1243 3750000
PostgreSQL-32-8-81920-4 PostgreSQL-32-8-81920 PostgreSQL-32-8-81920 1 0 lc56j.sensor 8 4 10240 30 a ... 284 284 284 3750000 1005.0607568 630 25804799 941 1161 3750000
PostgreSQL-32-8-81920-5 PostgreSQL-32-8-81920 PostgreSQL-32-8-81920 1 0 lccsk.sensor 8 4 10240 30 a ... 209 209 209 3750000 1023.3424666666666 631 25804799 943 1185 3750000
PostgreSQL-32-8-81920-6 PostgreSQL-32-8-81920 PostgreSQL-32-8-81920 1 0 nqq5s.sensor 8 4 10240 30 a ... 259 259 259 3750000 1071.9134168 682 25804799 1012 1253 3750000
PostgreSQL-32-8-81920-7 PostgreSQL-32-8-81920 PostgreSQL-32-8-81920 1 0 qpc6h.sensor 8 4 10240 30 a ... 196 196 196 3750000 1021.6099389333333 645 25804799 943 1173 3750000
PostgreSQL-32-8-81920-8 PostgreSQL-32-8-81920 PostgreSQL-32-8-81920 1 0 vtqn4.sensor 8 4 10240 30 a ... 217 217 217 3750000 1072.3814525333332 686 25804799 1009 1306 3750000
PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 1 0 6xx8q.sensor 8 4 12288 30 a ... 192 192 192 3750000 1045.2016928 632 23363583 1000 1229 3750000
PostgreSQL-32-8-98304-2 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 1 0 ghmf9.sensor 8 4 12288 30 a ... 200 200 200 3750000 1028.8221322666666 631 23363583 998 1251 3750000
PostgreSQL-32-8-98304-3 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 1 0 gqw9m.sensor 8 4 12288 30 a ... 195 195 195 3750000 1045.3850088 639 23363583 1011 1238 3750000
PostgreSQL-32-8-98304-4 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 1 0 jln48.sensor 8 4 12288 30 a ... 212 212 212 3750000 1015.8059450666667 636 23363583 919 1164 3750000
PostgreSQL-32-8-98304-5 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 1 0 n9xct.sensor 8 4 12288 30 a ... 237 237 237 3750000 1083.8865906666667 689 23363583 1001 1246 3750000
PostgreSQL-32-8-98304-6 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 1 0 p78x4.sensor 8 4 12288 30 a ... 287 287 287 3750000 1051.2896445333333 638 23363583 987 1220 3750000
PostgreSQL-32-8-98304-7 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 1 0 s4t2s.sensor 8 4 12288 30 a ... 234 234 234 3750000 991.6972021333333 638 23363583 914 1162 3750000
PostgreSQL-32-8-98304-8 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 1 0 zklhx.sensor 8 4 12288 30 a ... 260 260 260 3750000 1022.2611192 636 23363583 955 1198 3750000

72 rows × 36 columns

In [16]:
df.fillna(0, inplace=True)
df_plot = evaluation.loading_set_datatypes(df)

df_aggregated = evaluation.loading_aggregate_by_parallel_pods(df_plot)
df_aggregated.sort_values('target', inplace=True)
df_aggregated.T
Out[16]:
PostgreSQL-32-1-16384 PostgreSQL-32-8-16384 PostgreSQL-32-1-32768 PostgreSQL-32-8-32768 PostgreSQL-32-1-49152 PostgreSQL-32-8-49152 PostgreSQL-32-1-65536 PostgreSQL-32-8-65536 PostgreSQL-32-1-81920 PostgreSQL-32-8-81920 PostgreSQL-32-1-98304 PostgreSQL-32-8-98304 PostgreSQL-32-1-114688 PostgreSQL-32-8-114688 PostgreSQL-32-1-131072 PostgreSQL-32-8-131072
connection PostgreSQL-32-1-16384 PostgreSQL-32-8-16384 PostgreSQL-32-1-32768 PostgreSQL-32-8-32768 PostgreSQL-32-1-49152 PostgreSQL-32-8-49152 PostgreSQL-32-1-65536 PostgreSQL-32-8-65536 PostgreSQL-32-1-81920 PostgreSQL-32-8-81920 PostgreSQL-32-1-98304 PostgreSQL-32-8-98304 PostgreSQL-32-1-114688 PostgreSQL-32-8-114688 PostgreSQL-32-1-131072 PostgreSQL-32-8-131072
configuration PostgreSQL-32-1-16384 PostgreSQL-32-8-16384 PostgreSQL-32-1-32768 PostgreSQL-32-8-32768 PostgreSQL-32-1-49152 PostgreSQL-32-8-49152 PostgreSQL-32-1-65536 PostgreSQL-32-8-65536 PostgreSQL-32-1-81920 PostgreSQL-32-8-81920 PostgreSQL-32-1-98304 PostgreSQL-32-8-98304 PostgreSQL-32-1-114688 PostgreSQL-32-8-114688 PostgreSQL-32-1-131072 PostgreSQL-32-8-131072
experiment_run 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
client 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
pod mr62r.sensor 2hl8g.sensorgb9j5.sensorkjk85.sensormskcx.sensorpcmmg.sensorpmr6w.sensorqmfjw.sensorxrrbk.sensor 9fbk6.sensor 5m288.sensordxlcv.sensorj97qc.sensorlvlv2.sensornzswp.sensorqk7pg.sensorthvmb.sensorvbs9b.sensor gs5fb.sensor 7fnlv.sensor8pcnm.sensor9kkpj.sensorjlvdb.sensorm7bxp.sensors229l.sensorsb97n.sensorx2j79.sensor 6hqzb.sensor 42qlk.sensor5dqdh.sensor66dgh.sensor6778k.sensor6vllq.sensord8z5q.sensork8drx.sensorm4t2w.sensor nkd9n.sensor 2gmfh.sensor7nlbl.sensorb2bbf.sensorlc56j.sensorlccsk.sensornqq5s.sensorqpc6h.sensorvtqn4.sensor xzd7w.sensor 6xx8q.sensorghmf9.sensorgqw9m.sensorjln48.sensorn9xct.sensorp78x4.sensors4t2s.sensorzklhx.sensor tx5d5.sensor 5x62v.sensor7bc2c.sensorc7w5h.sensordg57m.sensordn5vx.sensorhkqrv.sensorqbf55.sensorsr6t8.sensor b9gzf.sensor 2dbvt.sensor2vfvb.sensor5qftv.sensorfwkgs.sensorn8gtm.sensorp6x48.sensorp9pwx.sensorvjc59.sensor
pod_count 1 8 1 8 1 8 1 8 1 8 1 8 1 8 1 8
threads 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32
target 16384 16384 32768 32768 49152 49152 65536 65536 81920 81920 98304 98304 114688 114688 131072 131072
sf 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30
workload a a a a a a a a a a a a a a a a
operations 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000
batchsize -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0
[OVERALL].RunTime(ms) 1831226.0 1831208.0 1059352.0 1070644.0 1062779.0 1050521.0 1030604.0 1092327.0 1054230.0 1073472.0 1074369.0 1069092.0 1066169.0 1124981.0 1085442.0 1090990.0
[OVERALL].Throughput(ops/sec) 16382.467265 16382.692041 28319.198907 28713.344934 28227.881808 29561.928753 29109.143764 28917.241547 28456.788367 29461.21192 27923.367111 29188.20774 28138.128195 29163.003574 27638.510395 29215.760305
[CLEANUP].Operations 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32
[CLEANUP].AverageLatency(us) 140.40625 117.09375 99.6875 153.375 90.25 129.1875 86.625 145.40625 95.25 130.4375 95.5 127.65625 95.4375 133.0 95.0 135.25
[CLEANUP].MinLatency(us) 32.0 33.0 53.0 60.0 47.0 56.0 55.0 57.0 54.0 58.0 54.0 59.0 57.0 60.0 37.0 60.0
[CLEANUP].MaxLatency(us) 692.0 423.0 195.0 506.0 199.0 373.0 206.0 436.0 214.0 284.0 375.0 287.0 186.0 448.0 199.0 368.0
[CLEANUP].95thPercentileLatency(us) 450.0 423.0 140.0 506.0 147.0 373.0 132.0 436.0 129.0 284.0 115.0 287.0 144.0 448.0 154.0 368.0
[CLEANUP].99thPercentileLatency(us) 692.0 423.0 195.0 506.0 199.0 373.0 206.0 436.0 214.0 284.0 375.0 287.0 186.0 448.0 199.0 368.0
[INSERT].Operations 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000
[INSERT].AverageLatency(us) 835.081132 836.357117 1019.092939 1014.948128 1042.662761 1036.62637 1032.856487 1033.149775 1039.479765 1036.791237 1031.23876 1035.543667 1024.359428 1044.506495 1033.995625 1027.836064
[INSERT].MinLatency(us) 633.0 637.0 630.0 634.0 630.0 630.0 629.0 630.0 636.0 630.0 630.0 631.0 629.0 629.0 636.0 632.0
[INSERT].MaxLatency(us) 385791.0 143231.0 26443775.0 25477119.0 23822335.0 23117823.0 23887871.0 22495231.0 23085055.0 25804799.0 23478271.0 23363583.0 18612223.0 23658495.0 24150015.0 22003711.0
[INSERT].95thPercentileLatency(us) 1025.0 1134.0 1018.0 1027.0 981.0 964.0 967.0 1105.0 967.0 1012.0 959.0 1011.0 967.0 1068.0 945.0 1077.0
[INSERT].99thPercentileLatency(us) 1180.0 1352.0 1261.0 1342.0 1268.0 1239.0 1255.0 1350.0 1261.0 1306.0 1263.0 1251.0 1261.0 1321.0 1268.0 1333.0
[INSERT].Return=OK 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000

Group by Connection¶

In [17]:
df_plot = evaluation.loading_set_datatypes(df)
df_groups = df_plot.groupby(['connection', 'pod'])

df_groups = df_groups.sum('target')
df_groups
Out[17]:
experiment_run client pod_count threads target sf operations batchsize [OVERALL].RunTime(ms) [OVERALL].Throughput(ops/sec) ... [CLEANUP].MaxLatency(us) [CLEANUP].95thPercentileLatency(us) [CLEANUP].99thPercentileLatency(us) [INSERT].Operations [INSERT].AverageLatency(us) [INSERT].MinLatency(us) [INSERT].MaxLatency(us) [INSERT].95thPercentileLatency(us) [INSERT].99thPercentileLatency(us) [INSERT].Return=OK
connection pod
PostgreSQL-32-1-114688 tx5d5.sensor 1 0 1 32 114688 30 30000000 -1 1066169.0 28138.128195 ... 186.0 144.0 186.0 30000000 1024.359428 629.0 18612223.0 967.0 1261.0 30000000
PostgreSQL-32-1-131072 b9gzf.sensor 1 0 1 32 131072 30 30000000 -1 1085442.0 27638.510395 ... 199.0 154.0 199.0 30000000 1033.995625 636.0 24150015.0 945.0 1268.0 30000000
PostgreSQL-32-1-16384 mr62r.sensor 1 0 1 32 16384 30 30000000 -1 1831226.0 16382.467265 ... 692.0 450.0 692.0 30000000 835.081132 633.0 385791.0 1025.0 1180.0 30000000
PostgreSQL-32-1-32768 9fbk6.sensor 1 0 1 32 32768 30 30000000 -1 1059352.0 28319.198907 ... 195.0 140.0 195.0 30000000 1019.092939 630.0 26443775.0 1018.0 1261.0 30000000
PostgreSQL-32-1-49152 gs5fb.sensor 1 0 1 32 49152 30 30000000 -1 1062779.0 28227.881808 ... 199.0 147.0 199.0 30000000 1042.662761 630.0 23822335.0 981.0 1268.0 30000000
PostgreSQL-32-1-65536 6hqzb.sensor 1 0 1 32 65536 30 30000000 -1 1030604.0 29109.143764 ... 206.0 132.0 206.0 30000000 1032.856487 629.0 23887871.0 967.0 1255.0 30000000
PostgreSQL-32-1-81920 nkd9n.sensor 1 0 1 32 81920 30 30000000 -1 1054230.0 28456.788367 ... 214.0 129.0 214.0 30000000 1039.479765 636.0 23085055.0 967.0 1261.0 30000000
PostgreSQL-32-1-98304 xzd7w.sensor 1 0 1 32 98304 30 30000000 -1 1074369.0 27923.367111 ... 375.0 115.0 375.0 30000000 1031.238760 630.0 23478271.0 959.0 1263.0 30000000
PostgreSQL-32-8-114688 5x62v.sensor 1 0 8 4 14336 30 3750000 -1 1032151.0 3633.189330 ... 187.0 187.0 187.0 3750000 1071.326950 653.0 23658495.0 1034.0 1278.0 3750000
7bc2c.sensor 1 0 8 4 14336 30 3750000 -1 1039143.0 3608.742974 ... 373.0 373.0 373.0 3750000 1064.048491 687.0 23658495.0 1015.0 1247.0 3750000
c7w5h.sensor 1 0 8 4 14336 30 3750000 -1 1124981.0 3333.389631 ... 359.0 359.0 359.0 3750000 1062.648087 698.0 23658495.0 1068.0 1321.0 3750000
dg57m.sensor 1 0 8 4 14336 30 3750000 -1 1018860.0 3680.584182 ... 202.0 202.0 202.0 3750000 1054.281335 639.0 23658495.0 988.0 1227.0 3750000
dn5vx.sensor 1 0 8 4 14336 30 3750000 -1 991561.0 3781.915586 ... 448.0 448.0 448.0 3750000 1046.161767 634.0 23658495.0 1002.0 1243.0 3750000
hkqrv.sensor 1 0 8 4 14336 30 3750000 -1 990135.0 3787.362329 ... 200.0 200.0 200.0 3750000 1015.111800 643.0 23658495.0 915.0 1154.0 3750000
qbf55.sensor 1 0 8 4 14336 30 3750000 -1 989158.0 3791.103140 ... 217.0 217.0 217.0 3750000 979.814381 629.0 23658495.0 930.0 1164.0 3750000
sr6t8.sensor 1 0 8 4 14336 30 3750000 -1 1057316.0 3546.716403 ... 195.0 195.0 195.0 3750000 1062.659146 682.0 23658495.0 1034.0 1260.0 3750000
PostgreSQL-32-8-131072 2dbvt.sensor 1 0 8 4 16384 30 3750000 -1 1019260.0 3679.139768 ... 214.0 214.0 214.0 3750000 1018.979515 640.0 22003711.0 961.0 1203.0 3750000
2vfvb.sensor 1 0 8 4 16384 30 3750000 -1 999208.0 3752.972354 ... 193.0 193.0 193.0 3750000 1037.851574 632.0 22003711.0 950.0 1194.0 3750000
5qftv.sensor 1 0 8 4 16384 30 3750000 -1 1085177.0 3455.657464 ... 368.0 368.0 368.0 3750000 1108.415739 642.0 22003711.0 1077.0 1333.0 3750000
fwkgs.sensor 1 0 8 4 16384 30 3750000 -1 1090990.0 3437.245071 ... 336.0 336.0 336.0 3750000 1011.058430 635.0 22003711.0 1018.0 1246.0 3750000
n8gtm.sensor 1 0 8 4 16384 30 3750000 -1 959749.0 3907.271589 ... 266.0 266.0 266.0 3750000 971.410110 636.0 22003711.0 851.0 1042.0 3750000
p6x48.sensor 1 0 8 4 16384 30 3750000 -1 1044507.0 3590.210501 ... 207.0 207.0 207.0 3750000 1026.584293 637.0 22003711.0 1017.0 1254.0 3750000
p9pwx.sensor 1 0 8 4 16384 30 3750000 -1 1016913.0 3687.631095 ... 211.0 211.0 211.0 3750000 989.811878 640.0 22003711.0 939.0 1149.0 3750000
vjc59.sensor 1 0 8 4 16384 30 3750000 -1 1011973.0 3705.632463 ... 279.0 279.0 279.0 3750000 1058.576972 636.0 22003711.0 965.0 1194.0 3750000
PostgreSQL-32-8-16384 2hl8g.sensor 1 0 8 4 2048 30 3750000 -1 1831205.0 2047.831892 ... 423.0 423.0 423.0 3750000 855.068822 643.0 143103.0 1063.0 1201.0 3750000
gb9j5.sensor 1 0 8 4 2048 30 3750000 -1 1831208.0 2047.828537 ... 214.0 214.0 214.0 3750000 832.772679 647.0 142975.0 1025.0 1129.0 3750000
kjk85.sensor 1 0 8 4 2048 30 3750000 -1 1831207.0 2047.829656 ... 187.0 187.0 187.0 3750000 812.780115 646.0 142975.0 977.0 1089.0 3750000
mskcx.sensor 1 0 8 4 2048 30 3750000 -1 1831193.0 2047.845312 ... 188.0 188.0 188.0 3750000 839.590646 637.0 143103.0 1034.0 1220.0 3750000
pcmmg.sensor 1 0 8 4 2048 30 3750000 -1 1831201.0 2047.836365 ... 191.0 191.0 191.0 3750000 790.366524 646.0 142975.0 973.0 1044.0 3750000
pmr6w.sensor 1 0 8 4 2048 30 3750000 -1 1831203.0 2047.834129 ... 186.0 186.0 186.0 3750000 856.366943 643.0 143231.0 1107.0 1275.0 3750000
qmfjw.sensor 1 0 8 4 2048 30 3750000 -1 1831193.0 2047.845312 ... 251.0 251.0 251.0 3750000 893.798743 650.0 143231.0 1134.0 1352.0 3750000
xrrbk.sensor 1 0 8 4 2048 30 3750000 -1 1831197.0 2047.840839 ... 186.0 186.0 186.0 3750000 810.112466 649.0 143103.0 1000.0 1213.0 3750000
PostgreSQL-32-8-32768 5m288.sensor 1 0 8 4 4096 30 3750000 -1 1070644.0 3502.564811 ... 506.0 506.0 506.0 3750000 1017.997042 635.0 25477119.0 1027.0 1298.0 3750000
dxlcv.sensor 1 0 8 4 4096 30 3750000 -1 1038352.0 3611.492057 ... 217.0 217.0 217.0 3750000 1018.753053 642.0 25477119.0 991.0 1275.0 3750000
j97qc.sensor 1 0 8 4 4096 30 3750000 -1 1028056.0 3647.661217 ... 190.0 190.0 190.0 3750000 1013.766474 641.0 25477119.0 987.0 1317.0 3750000
lvlv2.sensor 1 0 8 4 4096 30 3750000 -1 1046118.0 3584.681652 ... 470.0 470.0 470.0 3750000 1008.154887 634.0 25477119.0 984.0 1248.0 3750000
nzswp.sensor 1 0 8 4 4096 30 3750000 -1 1063681.0 3525.493075 ... 192.0 192.0 192.0 3750000 1050.261742 641.0 25477119.0 1024.0 1342.0 3750000
qk7pg.sensor 1 0 8 4 4096 30 3750000 -1 1020079.0 3676.185864 ... 204.0 204.0 204.0 3750000 982.031746 635.0 25477119.0 959.0 1140.0 3750000
thvmb.sensor 1 0 8 4 4096 30 3750000 -1 1047311.0 3580.598313 ... 194.0 194.0 194.0 3750000 1003.515115 634.0 25477119.0 993.0 1229.0 3750000
vbs9b.sensor 1 0 8 4 4096 30 3750000 -1 1046122.0 3584.667945 ... 474.0 474.0 474.0 3750000 1025.104966 643.0 25477119.0 1007.0 1229.0 3750000
PostgreSQL-32-8-49152 7fnlv.sensor 1 0 8 4 6144 30 3750000 -1 1011511.0 3707.324982 ... 373.0 373.0 373.0 3750000 1041.566758 636.0 23117823.0 938.0 1178.0 3750000
8pcnm.sensor 1 0 8 4 6144 30 3750000 -1 979584.0 3828.155625 ... 213.0 213.0 213.0 3750000 1032.728345 640.0 23101439.0 889.0 1153.0 3750000
9kkpj.sensor 1 0 8 4 6144 30 3750000 -1 1050521.0 3569.657341 ... 189.0 189.0 189.0 3750000 1058.016423 640.0 23117823.0 946.0 1239.0 3750000
jlvdb.sensor 1 0 8 4 6144 30 3750000 -1 1023677.0 3663.264877 ... 190.0 190.0 190.0 3750000 1018.256093 636.0 23101439.0 912.0 1162.0 3750000
m7bxp.sensor 1 0 8 4 6144 30 3750000 -1 985892.0 3803.662064 ... 198.0 198.0 198.0 3750000 1020.956194 633.0 23101439.0 862.0 1116.0 3750000
s229l.sensor 1 0 8 4 6144 30 3750000 -1 1046148.0 3584.578855 ... 210.0 210.0 210.0 3750000 1035.592908 639.0 23101439.0 930.0 1207.0 3750000
sb97n.sensor 1 0 8 4 6144 30 3750000 -1 1038195.0 3612.038201 ... 222.0 222.0 222.0 3750000 1074.382773 649.0 23101439.0 964.0 1237.0 3750000
x2j79.sensor 1 0 8 4 6144 30 3750000 -1 988599.0 3793.246807 ... 205.0 205.0 205.0 3750000 1011.511462 630.0 23101439.0 859.0 1105.0 3750000
PostgreSQL-32-8-65536 42qlk.sensor 1 0 8 4 8192 30 3750000 -1 1015387.0 3693.173145 ... 210.0 210.0 210.0 3750000 996.883223 630.0 22495231.0 990.0 1205.0 3750000
5dqdh.sensor 1 0 8 4 8192 30 3750000 -1 1035542.0 3621.292038 ... 404.0 404.0 404.0 3750000 1075.093785 702.0 22495231.0 1048.0 1269.0 3750000
66dgh.sensor 1 0 8 4 8192 30 3750000 -1 981090.0 3822.279302 ... 436.0 436.0 436.0 3750000 983.916262 637.0 22495231.0 942.0 1171.0 3750000
6778k.sensor 1 0 8 4 8192 30 3750000 -1 1070699.0 3502.384891 ... 373.0 373.0 373.0 3750000 1075.245356 651.0 22495231.0 1070.0 1293.0 3750000
6vllq.sensor 1 0 8 4 8192 30 3750000 -1 1092327.0 3433.037909 ... 354.0 354.0 354.0 3750000 1086.000919 650.0 22495231.0 1105.0 1350.0 3750000
d8z5q.sensor 1 0 8 4 8192 30 3750000 -1 1019868.0 3676.946428 ... 251.0 251.0 251.0 3750000 1005.459416 642.0 22495231.0 970.0 1191.0 3750000
k8drx.sensor 1 0 8 4 8192 30 3750000 -1 1037218.0 3615.440534 ... 230.0 230.0 230.0 3750000 987.982993 635.0 22495231.0 985.0 1216.0 3750000
m4t2w.sensor 1 0 8 4 8192 30 3750000 -1 1055539.0 3552.687300 ... 210.0 210.0 210.0 3750000 1054.616247 696.0 22495231.0 1030.0 1261.0 3750000
PostgreSQL-32-8-81920 2gmfh.sensor 1 0 8 4 10240 30 3750000 -1 1009671.0 3714.081121 ... 205.0 205.0 205.0 3750000 1040.131713 630.0 25804799.0 943.0 1197.0 3750000
7nlbl.sensor 1 0 8 4 10240 30 3750000 -1 984110.0 3810.549634 ... 221.0 221.0 221.0 3750000 995.793029 632.0 25804799.0 901.0 1142.0 3750000
b2bbf.sensor 1 0 8 4 10240 30 3750000 -1 1029558.0 3642.339722 ... 185.0 185.0 185.0 3750000 1064.097122 686.0 25804799.0 996.0 1243.0 3750000
lc56j.sensor 1 0 8 4 10240 30 3750000 -1 1013965.0 3698.352507 ... 284.0 284.0 284.0 3750000 1005.060757 630.0 25804799.0 941.0 1161.0 3750000
lccsk.sensor 1 0 8 4 10240 30 3750000 -1 992998.0 3776.442651 ... 209.0 209.0 209.0 3750000 1023.342467 631.0 25804799.0 943.0 1185.0 3750000
nqq5s.sensor 1 0 8 4 10240 30 3750000 -1 1043412.0 3593.978218 ... 259.0 259.0 259.0 3750000 1071.913417 682.0 25804799.0 1012.0 1253.0 3750000
qpc6h.sensor 1 0 8 4 10240 30 3750000 -1 1004788.0 3732.130559 ... 196.0 196.0 196.0 3750000 1021.609939 645.0 25804799.0 943.0 1173.0 3750000
vtqn4.sensor 1 0 8 4 10240 30 3750000 -1 1073472.0 3493.337507 ... 217.0 217.0 217.0 3750000 1072.381453 686.0 25804799.0 1009.0 1306.0 3750000
PostgreSQL-32-8-98304 6xx8q.sensor 1 0 8 4 12288 30 3750000 -1 1054586.0 3555.897765 ... 192.0 192.0 192.0 3750000 1045.201693 632.0 23363583.0 1000.0 1229.0 3750000
ghmf9.sensor 1 0 8 4 12288 30 3750000 -1 1069092.0 3507.649482 ... 200.0 200.0 200.0 3750000 1028.822132 631.0 23363583.0 998.0 1251.0 3750000
gqw9m.sensor 1 0 8 4 12288 30 3750000 -1 1054273.0 3556.953465 ... 195.0 195.0 195.0 3750000 1045.385009 639.0 23363583.0 1011.0 1238.0 3750000
jln48.sensor 1 0 8 4 12288 30 3750000 -1 979125.0 3829.950211 ... 212.0 212.0 212.0 3750000 1015.805945 636.0 23363583.0 919.0 1164.0 3750000
n9xct.sensor 1 0 8 4 12288 30 3750000 -1 1054125.0 3557.452864 ... 237.0 237.0 237.0 3750000 1083.886591 689.0 23363583.0 1001.0 1246.0 3750000
p78x4.sensor 1 0 8 4 12288 30 3750000 -1 1013255.0 3700.943987 ... 287.0 287.0 287.0 3750000 1051.289645 638.0 23363583.0 987.0 1220.0 3750000
s4t2s.sensor 1 0 8 4 12288 30 3750000 -1 970744.0 3863.016408 ... 234.0 234.0 234.0 3750000 991.697202 638.0 23363583.0 914.0 1162.0 3750000
zklhx.sensor 1 0 8 4 12288 30 3750000 -1 1036959.0 3616.343558 ... 260.0 260.0 260.0 3750000 1022.261119 636.0 23363583.0 955.0 1198.0 3750000

72 rows × 23 columns

Get Maximum Number of Parallel Pods¶

In [18]:
max_pod_count = df_groups['pod_count'].max()
max_pod_count
Out[18]:
8

Plot Varations¶

In [19]:
y='[INSERT].MinLatency(us)'
plot_variation(df_groups, max_pod_count, y)
No description has been provided for this image
In [20]:
y='[INSERT].MaxLatency(us)'
plot_variation(df_groups, max_pod_count, y)
No description has been provided for this image
In [21]:
y='[INSERT].AverageLatency(us)'
plot_variation(df_groups, max_pod_count, y)
No description has been provided for this image
In [22]:
y='[INSERT].99thPercentileLatency(us)'
plot_variation(df_groups, max_pod_count, y)
No description has been provided for this image
In [23]:
y='[INSERT].95thPercentileLatency(us)'
plot_variation(df_groups, max_pod_count, y)
No description has been provided for this image
In [24]:
y='[OVERALL].Throughput(ops/sec)'
plot_variation(df_groups, max_pod_count, y)
No description has been provided for this image
In [25]:
by='target'
column='[OVERALL].Throughput(ops/sec)'
groups = df_groups.groupby('pod_count')
rows = 1#(len(groups)+1)//2
row=0
col=0
fig, axes = plt.subplots(nrows=rows, ncols=2, squeeze=False, figsize=(6,4*rows))#, sharex=True
for key1, grp in groups:#df3.groupby(col1):
    ax = grp.boxplot(ax=axes[row,col], by='target', column='[OVERALL].Throughput(ops/sec)', figsize=(6,4), layout=(rows,2), rot=90)
    col = col + 1
    if col > 1:
        row = row + 1
        col = 0
plt.legend(loc='best')
plt.tight_layout()
plt.show()

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
No description has been provided for this image

Plot Comparison¶

In [26]:
column = "pods"
x = "target"
y = "[OVERALL].Throughput(ops/sec)"
title = "Throughput [ops/s]"
plot_comparison(df_aggregated, x, y, column, title, peak_loading)
No description has been provided for this image
In [27]:
column = "pods"
x = "target"
y = "[INSERT].AverageLatency(us)"
title = "Avg Lat [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_loading)
No description has been provided for this image
In [28]:
column = "pods"
x = "target"
y = "[INSERT].95thPercentileLatency(us)"
title = "95th Lat [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_loading)
No description has been provided for this image
In [29]:
column = "pods"
x = "target"
y = "[INSERT].99thPercentileLatency(us)"
title = "99th Lat [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_loading)
No description has been provided for this image
In [30]:
column = "pods"
x = "target"
y = "[INSERT].MaxLatency(us)"
y2 = "[INSERT].MaxLatency(ms)"
title = "Max Lat [ms]"
df_aggregated[y2] = df_aggregated[y]/1000.
plot_comparison(df_aggregated, x, y2, column, title, peak_loading)
No description has been provided for this image
In [31]:
column = "pods"
x = "target"
y = "[INSERT].MinLatency(us)"
title = "Min Lat [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_loading)
No description has been provided for this image

Get Benchmarking Result¶

In [32]:
df = evaluation.get_df_benchmarking()
#df = df[df.columns.drop(list(df.filter(regex='FAILED')))]
df.T
Out[32]:
connection_pod PostgreSQL-32-1-114688-1-1 PostgreSQL-32-1-131072-1-1 PostgreSQL-32-1-16384-1-1 PostgreSQL-32-1-32768-1-1 PostgreSQL-32-1-49152-1-1 PostgreSQL-32-1-65536-1-1 PostgreSQL-32-1-81920-1-1 PostgreSQL-32-1-98304-1-1 PostgreSQL-32-8-114688-1-1 PostgreSQL-32-8-114688-1-2 ... PostgreSQL-32-8-81920-1-7 PostgreSQL-32-8-81920-1-8 PostgreSQL-32-8-98304-1-1 PostgreSQL-32-8-98304-1-2 PostgreSQL-32-8-98304-1-3 PostgreSQL-32-8-98304-1-4 PostgreSQL-32-8-98304-1-5 PostgreSQL-32-8-98304-1-6 PostgreSQL-32-8-98304-1-7 PostgreSQL-32-8-98304-1-8
connection PostgreSQL-32-1-114688-1 PostgreSQL-32-1-131072-1 PostgreSQL-32-1-16384-1 PostgreSQL-32-1-32768-1 PostgreSQL-32-1-49152-1 PostgreSQL-32-1-65536-1 PostgreSQL-32-1-81920-1 PostgreSQL-32-1-98304-1 PostgreSQL-32-8-114688-1 PostgreSQL-32-8-114688-1 ... PostgreSQL-32-8-81920-1 PostgreSQL-32-8-81920-1 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304-1
configuration PostgreSQL-32-1-114688 PostgreSQL-32-1-131072 PostgreSQL-32-1-16384 PostgreSQL-32-1-32768 PostgreSQL-32-1-49152 PostgreSQL-32-1-65536 PostgreSQL-32-1-81920 PostgreSQL-32-1-98304 PostgreSQL-32-8-114688 PostgreSQL-32-8-114688 ... PostgreSQL-32-8-81920 PostgreSQL-32-8-81920 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304 PostgreSQL-32-8-98304
experiment_run 1 1 1 1 1 1 1 1 1 1 ... 1 1 1 1 1 1 1 1 1 1
client 1 1 1 1 1 1 1 1 1 1 ... 1 1 1 1 1 1 1 1 1 1
pod mwnxg zp9vj dpz6r k6vs7 klpvs 7w4z9 chmc5 7lqx2 2jq82 8tw89 ... vhthf xn2w5 45zhr bhvd9 k4r5r kq9kf mlt4b rksk7 vn5b4 wjjww
pod_count 1 1 1 1 1 1 1 1 8 8 ... 8 8 8 8 8 8 8 8 8 8
threads 32 32 32 32 32 32 32 32 4 4 ... 4 4 4 4 4 4 4 4 4 4
target 114688 131072 16384 32768 49152 65536 81920 98304 14336 14336 ... 10240 10240 12288 12288 12288 12288 12288 12288 12288 12288
sf 30 30 30 30 30 30 30 30 30 30 ... 30 30 30 30 30 30 30 30 30 30
workload a a a a a a a a a a ... a a a a a a a a a a
operations 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 3750000 3750000 ... 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000 3750000
batchsize -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 ... -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
[OVERALL].RunTime(ms) 515631 521364 1831314 915761 657683 523264 521007 514630 520977 522245 ... 515641 532786 517972 525430 510878 509697 523738 513476 523770 513732
[OVERALL].Throughput(ops/sec) 58181.14116490281 57541.37224664534 16381.68003957814 32759.63925085257 45614.68062881358 57332.43639921722 57580.800257961986 58294.308532343624 7198.014499680408 7180.537870156727 ... 7272.501604798687 7038.473233155526 7239.773578494591 7137.011590506823 7340.304338804959 7357.312285534347 7160.068583910276 7303.165094376368 7159.631135803884 7299.5258228025505
[TOTAL_GCS_Copy].Count 528 531 526 533 531 530 531 534 66 65 ... 64 65 66 66 66 66 66 66 65 66
[TOTAL_GC_TIME_Copy].Time(ms) 773 766 736 738 779 763 765 776 251 200 ... 310 232 229 202 280 224 186 331 295 302
[TOTAL_GC_TIME_%_Copy].Time(%) 0.1499134070682329 0.14692230380310112 0.04018972169709837 0.08058871255709732 0.11844612069948592 0.14581549657534246 0.14683104065780306 0.1507879447369955 0.048178710384527534 0.03829620197416921 ... 0.06011934659966915 0.04354468773578885 0.04421088398600696 0.03844470243419675 0.0548076057297437 0.043947678718925166 0.03551394017619497 0.06446260389969541 0.056322431601657216 0.0587855146263032
[TOTAL_GCS_MarkSweepCompact].Count 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
[TOTAL_GC_TIME_MarkSweepCompact].Time(ms) 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
[TOTAL_GC_TIME_%_MarkSweepCompact].Time(%) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
[TOTAL_GCs].Count 528 531 526 533 531 530 531 534 66 65 ... 64 65 66 66 66 66 66 66 65 66
[TOTAL_GC_TIME].Time(ms) 773 766 736 738 779 763 765 776 251 200 ... 310 232 229 202 280 224 186 331 295 302
[TOTAL_GC_TIME_%].Time(%) 0.1499134070682329 0.14692230380310112 0.04018972169709837 0.08058871255709732 0.11844612069948592 0.14581549657534246 0.14683104065780306 0.1507879447369955 0.048178710384527534 0.03829620197416921 ... 0.06011934659966915 0.04354468773578885 0.04421088398600696 0.03844470243419675 0.0548076057297437 0.043947678718925166 0.03551394017619497 0.06446260389969541 0.056322431601657216 0.0587855146263032
[READ].Operations 14999034 15000437 15000202 14999278 15001696 15003790 14998146 15004063 1875496 1872886 ... 1874668 1874136 1875638 1875215 1876860 1873827 1876454 1873752 1874515 1875658
[READ].AverageLatency(us) 414.1533992122426 418.6445766213344 236.30964003018093 244.15625558776895 349.14303242780016 419.17214543791937 432.82956566764983 424.96967601375707 448.51290112055693 477.0497755869818 ... 432.95185760892065 478.3749418398665 469.7713530009522 451.09467714368753 470.12151732148374 437.6148956120282 440.84926622235344 416.91854151456545 431.3523418057471 426.3018178154013
[READ].MinLatency(us) 86 86 93 91 89 89 85 86 95 91 ... 101 91 98 91 96 106 91 95 94 96
[READ].MaxLatency(us) 20905983 20856831 70271 1073151 18399231 18235391 20709375 18776063 13819903 13819903 ... 13230079 13942783 13852671 12337151 18415615 17629183 13852671 16990207 16728063 9420799
[READ].95thPercentileLatency(us) 762 750 392 395 482 774 788 757 768 850 ... 790 921 769 861 787 723 757 696 779 772
[READ].99thPercentileLatency(us) 2147 2287 514 540 1209 2289 2355 2301 2435 2401 ... 2305 2245 2481 2355 2463 2377 2327 2191 2543 2399
[READ].Return=OK 14999034 15000437 15000202 14999278 15001696 15003790 14998146 15004063 1875496 1872886 ... 1874668 1874136 1875638 1875215 1876860 1873827 1876454 1873752 1874515 1875658
[CLEANUP].Operations 32 32 32 32 32 32 32 32 4 4 ... 4 4 4 4 4 4 4 4 4 4
[CLEANUP].AverageLatency(us) 104.75 97.0625 68.09375 100.96875 174.59375 122.875 138.46875 126.4375 224.75 143.75 ... 195.75 135.5 156.75 143.5 141.25 148.25 458.0 182.0 318.75 141.75
[CLEANUP].MinLatency(us) 60 62 35 36 55 70 64 64 65 70 ... 116 65 76 67 78 87 74 96 61 77
[CLEANUP].MaxLatency(us) 305 305 269 651 2655 343 938 377 420 352 ... 320 335 358 352 304 270 1324 355 991 299
[CLEANUP].95thPercentileLatency(us) 178 132 170 343 146 168 175 187 420 352 ... 320 335 358 352 304 270 1324 355 991 299
[CLEANUP].99thPercentileLatency(us) 305 305 269 651 2655 343 938 377 420 352 ... 320 335 358 352 304 270 1324 355 991 299
[UPDATE].Operations 15000966 14999563 14999798 15000722 14998304 14996210 15001854 14995937 1874504 1877114 ... 1875332 1875864 1874362 1874785 1873140 1876173 1873546 1876248 1875485 1874342
[UPDATE].AverageLatency(us) 635.8843409151117 647.1705313014786 260.75799487433096 262.927870671825 454.03203375528324 644.1248997580055 630.4081144237239 620.0819837399957 569.8579874996266 599.2677647708131 ... 621.9825412247005 618.4722176021289 610.3075121027848 620.0627383940025 592.218151873325 606.3485243631584 621.7419929908312 604.16092648733 621.6882539716394 640.6968872276244
[UPDATE].MinLatency(us) 95 94 102 101 93 93 93 93 99 96 ... 105 98 105 98 101 114 99 107 102 104
[UPDATE].MaxLatency(us) 20987903 20889599 67071 722943 18235391 20774911 20758527 18956287 20135935 18825215 ... 20430847 20955135 18710527 18956287 18956287 18317311 17874943 18776063 18759679 18579455
[UPDATE].95thPercentileLatency(us) 1041 1061 434 415 535 1075 1115 1065 1010 1047 ... 1007 1088 1032 1086 1072 1011 1015 966 1080 1043
[UPDATE].99thPercentileLatency(us) 3383 3619 573 580 1687 3657 3731 3641 3501 3589 ... 3367 3275 3517 3495 3579 3465 3449 3267 3613 3525
[UPDATE].Return=OK 15000966 14999563 14999798 15000722 14998304 14996210 15001854 14995937 1874504 1877114 ... 1875332 1875864 1874362 1874785 1873140 1876173 1873546 1876248 1875485 1874342

43 rows × 72 columns

Reconstruct Workflow out of Result¶

How many benchmarker have been used per configuration?

In [33]:
evaluation.reconstruct_workflow(df)
Out[33]:
{'PostgreSQL-32-1-114688': [[1]],
 'PostgreSQL-32-1-131072': [[1]],
 'PostgreSQL-32-1-16384': [[1]],
 'PostgreSQL-32-1-32768': [[1]],
 'PostgreSQL-32-1-49152': [[1]],
 'PostgreSQL-32-1-65536': [[1]],
 'PostgreSQL-32-1-81920': [[1]],
 'PostgreSQL-32-1-98304': [[1]],
 'PostgreSQL-32-8-114688': [[8]],
 'PostgreSQL-32-8-131072': [[8]],
 'PostgreSQL-32-8-16384': [[8]],
 'PostgreSQL-32-8-32768': [[8]],
 'PostgreSQL-32-8-49152': [[8]],
 'PostgreSQL-32-8-65536': [[8]],
 'PostgreSQL-32-8-81920': [[8]],
 'PostgreSQL-32-8-98304': [[8]]}

Plot Results¶

Set Data Types for Plotting¶

In [34]:
df.fillna(0, inplace=True)
#df.drop(df[df['[READ].Operations'] == 0].index, inplace=True)
#df.drop(df[df['[READ].Return=OK'] == 0].index, inplace=True)

df_plot = evaluation.benchmarking_set_datatypes(df)
df_plot.sort_values('target', inplace=True)
df_plot
Out[34]:
connection configuration experiment_run client pod pod_count threads target sf workload ... [CLEANUP].MaxLatency(us) [CLEANUP].95thPercentileLatency(us) [CLEANUP].99thPercentileLatency(us) [UPDATE].Operations [UPDATE].AverageLatency(us) [UPDATE].MinLatency(us) [UPDATE].MaxLatency(us) [UPDATE].95thPercentileLatency(us) [UPDATE].99thPercentileLatency(us) [UPDATE].Return=OK
connection_pod
PostgreSQL-32-8-16384-1-2 PostgreSQL-32-8-16384-1 PostgreSQL-32-8-16384 1 1 6qp9s 8 4 2048 30 a ... 267.0 267.0 267.0 1875349 279.281535 111.0 130815.0 430.0 692.0 1875349
PostgreSQL-32-8-16384-1-3 PostgreSQL-32-8-16384-1 PostgreSQL-32-8-16384 1 1 77mw4 8 4 2048 30 a ... 228.0 228.0 228.0 1874880 279.843256 109.0 77247.0 503.0 648.0 1874880
PostgreSQL-32-8-16384-1-4 PostgreSQL-32-8-16384-1 PostgreSQL-32-8-16384 1 1 cpwg6 8 4 2048 30 a ... 244.0 244.0 244.0 1875427 269.904560 109.0 130175.0 415.0 649.0 1875427
PostgreSQL-32-8-16384-1-5 PostgreSQL-32-8-16384-1 PostgreSQL-32-8-16384 1 1 cxgj6 8 4 2048 30 a ... 204.0 204.0 204.0 1875337 297.950520 109.0 76415.0 467.0 640.0 1875337
PostgreSQL-32-8-16384-1-6 PostgreSQL-32-8-16384-1 PostgreSQL-32-8-16384 1 1 jpmmp 8 4 2048 30 a ... 292.0 292.0 292.0 1875014 261.791274 108.0 55775.0 398.0 625.0 1875014
PostgreSQL-32-8-16384-1-7 PostgreSQL-32-8-16384-1 PostgreSQL-32-8-16384 1 1 rt4r9 8 4 2048 30 a ... 230.0 230.0 230.0 1873638 291.152009 114.0 118847.0 515.0 655.0 1873638
PostgreSQL-32-8-16384-1-8 PostgreSQL-32-8-16384-1 PostgreSQL-32-8-16384 1 1 tfbqm 8 4 2048 30 a ... 215.0 215.0 215.0 1875605 285.543071 113.0 127295.0 491.0 633.0 1875605
PostgreSQL-32-8-16384-1-1 PostgreSQL-32-8-16384-1 PostgreSQL-32-8-16384 1 1 28mm2 8 4 2048 30 a ... 238.0 238.0 238.0 1873921 259.960231 108.0 88511.0 461.0 614.0 1873921
PostgreSQL-32-8-32768-1-4 PostgreSQL-32-8-32768-1 PostgreSQL-32-8-32768 1 1 j2j7f 8 4 4096 30 a ... 324.0 324.0 324.0 1876703 312.227578 107.0 735231.0 478.0 929.0 1876703
PostgreSQL-32-8-32768-1-1 PostgreSQL-32-8-32768-1 PostgreSQL-32-8-32768 1 1 2pf8r 8 4 4096 30 a ... 282.0 282.0 282.0 1874112 290.050854 109.0 97407.0 454.0 946.0 1874112
PostgreSQL-32-8-32768-1-2 PostgreSQL-32-8-32768-1 PostgreSQL-32-8-32768 1 1 5szs7 8 4 4096 30 a ... 236.0 236.0 236.0 1875046 295.300415 110.0 332287.0 437.0 1008.0 1875046
PostgreSQL-32-8-32768-1-3 PostgreSQL-32-8-32768-1 PostgreSQL-32-8-32768 1 1 cfgn7 8 4 4096 30 a ... 260.0 260.0 260.0 1874994 304.605591 103.0 1277951.0 471.0 971.0 1874994
PostgreSQL-32-8-32768-1-5 PostgreSQL-32-8-32768-1 PostgreSQL-32-8-32768 1 1 lc8b4 8 4 4096 30 a ... 264.0 264.0 264.0 1873702 282.308903 105.0 159871.0 430.0 929.0 1873702
PostgreSQL-32-8-32768-1-6 PostgreSQL-32-8-32768-1 PostgreSQL-32-8-32768 1 1 mt7d6 8 4 4096 30 a ... 323.0 323.0 323.0 1874082 309.469422 110.0 86015.0 508.0 823.0 1874082
PostgreSQL-32-8-32768-1-7 PostgreSQL-32-8-32768-1 PostgreSQL-32-8-32768 1 1 p9tms 8 4 4096 30 a ... 301.0 301.0 301.0 1875937 312.520923 109.0 539135.0 485.0 975.0 1875937
PostgreSQL-32-8-32768-1-8 PostgreSQL-32-8-32768-1 PostgreSQL-32-8-32768 1 1 rt62c 8 4 4096 30 a ... 344.0 344.0 344.0 1876930 313.031167 111.0 270079.0 482.0 971.0 1876930
PostgreSQL-32-8-49152-1-1 PostgreSQL-32-8-49152-1 PostgreSQL-32-8-49152 1 1 5wh9k 8 4 6144 30 a ... 349.0 349.0 349.0 1875039 483.460633 105.0 12902399.0 625.0 2353.0 1875039
PostgreSQL-32-8-49152-1-2 PostgreSQL-32-8-49152-1 PostgreSQL-32-8-49152 1 1 998gf 8 4 6144 30 a ... 278.0 278.0 278.0 1874709 473.292028 101.0 18120703.0 637.0 2247.0 1874709
PostgreSQL-32-8-49152-1-3 PostgreSQL-32-8-49152-1 PostgreSQL-32-8-49152 1 1 ghk57 8 4 6144 30 a ... 425.0 425.0 425.0 1873652 469.840329 105.0 16007167.0 651.0 2293.0 1873652
PostgreSQL-32-8-49152-1-4 PostgreSQL-32-8-49152-1 PostgreSQL-32-8-49152 1 1 rvqs4 8 4 6144 30 a ... 1652.0 1652.0 1652.0 1875591 507.224280 105.0 18219007.0 658.0 2231.0 1875591
PostgreSQL-32-8-49152-1-7 PostgreSQL-32-8-49152-1 PostgreSQL-32-8-49152 1 1 tqqdc 8 4 6144 30 a ... 307.0 307.0 307.0 1874986 494.877304 101.0 14909439.0 675.0 2257.0 1874986
PostgreSQL-32-8-49152-1-8 PostgreSQL-32-8-49152-1 PostgreSQL-32-8-49152 1 1 xwgtc 8 4 6144 30 a ... 348.0 348.0 348.0 1876064 441.921570 105.0 14909439.0 646.0 2269.0 1876064
PostgreSQL-32-8-49152-1-6 PostgreSQL-32-8-49152-1 PostgreSQL-32-8-49152 1 1 sgxpf 8 4 6144 30 a ... 311.0 311.0 311.0 1874523 464.176237 107.0 18382847.0 641.0 2301.0 1874523
PostgreSQL-32-8-49152-1-5 PostgreSQL-32-8-49152-1 PostgreSQL-32-8-49152 1 1 sfzq5 8 4 6144 30 a ... 304.0 304.0 304.0 1874898 494.085958 107.0 21233663.0 654.0 2133.0 1874898
PostgreSQL-32-8-65536-1-2 PostgreSQL-32-8-65536-1 PostgreSQL-32-8-65536 1 1 l25nl 8 4 8192 30 a ... 301.0 301.0 301.0 1875272 620.948668 101.0 20856831.0 1096.0 3859.0 1875272
PostgreSQL-32-8-65536-1-8 PostgreSQL-32-8-65536-1 PostgreSQL-32-8-65536 1 1 z58tr 8 4 8192 30 a ... 290.0 290.0 290.0 1872314 603.486167 107.0 13221887.0 1076.0 3917.0 1872314
PostgreSQL-32-8-65536-1-7 PostgreSQL-32-8-65536-1 PostgreSQL-32-8-65536 1 1 wp69b 8 4 8192 30 a ... 322.0 322.0 322.0 1874271 618.117459 103.0 20905983.0 1085.0 3907.0 1874271
PostgreSQL-32-8-65536-1-1 PostgreSQL-32-8-65536-1 PostgreSQL-32-8-65536 1 1 5spjq 8 4 8192 30 a ... 313.0 313.0 313.0 1875000 634.823838 100.0 21069823.0 1046.0 3931.0 1875000
PostgreSQL-32-8-65536-1-5 PostgreSQL-32-8-65536-1 PostgreSQL-32-8-65536 1 1 rh8zl 8 4 8192 30 a ... 515.0 515.0 515.0 1875343 610.272305 100.0 20627455.0 1018.0 3753.0 1875343
PostgreSQL-32-8-65536-1-4 PostgreSQL-32-8-65536-1 PostgreSQL-32-8-65536 1 1 nws4x 8 4 8192 30 a ... 332.0 332.0 332.0 1875071 622.472328 104.0 16457727.0 1053.0 3851.0 1875071
PostgreSQL-32-8-65536-1-6 PostgreSQL-32-8-65536-1 PostgreSQL-32-8-65536 1 1 vc88w 8 4 8192 30 a ... 608.0 608.0 608.0 1875539 628.395195 102.0 20987903.0 1081.0 3865.0 1875539
PostgreSQL-32-8-65536-1-3 PostgreSQL-32-8-65536-1 PostgreSQL-32-8-65536 1 1 lrgp7 8 4 8192 30 a ... 297.0 297.0 297.0 1875517 609.462749 103.0 20922367.0 1087.0 3897.0 1875517
PostgreSQL-32-8-81920-1-8 PostgreSQL-32-8-81920-1 PostgreSQL-32-8-81920 1 1 xn2w5 8 4 10240 30 a ... 335.0 335.0 335.0 1875864 618.472218 98.0 20955135.0 1088.0 3275.0 1875864
PostgreSQL-32-8-81920-1-6 PostgreSQL-32-8-81920-1 PostgreSQL-32-8-81920 1 1 tt4fm 8 4 10240 30 a ... 362.0 362.0 362.0 1875823 597.406558 113.0 20463615.0 972.0 3363.0 1875823
PostgreSQL-32-8-81920-1-7 PostgreSQL-32-8-81920-1 PostgreSQL-32-8-81920 1 1 vhthf 8 4 10240 30 a ... 320.0 320.0 320.0 1875332 621.982541 105.0 20430847.0 1007.0 3367.0 1875332
PostgreSQL-32-8-81920-1-3 PostgreSQL-32-8-81920-1 PostgreSQL-32-8-81920 1 1 b7bh7 8 4 10240 30 a ... 384.0 384.0 384.0 1873757 583.579925 105.0 20496383.0 951.0 3315.0 1873757
PostgreSQL-32-8-81920-1-2 PostgreSQL-32-8-81920-1 PostgreSQL-32-8-81920 1 1 9wm6b 8 4 10240 30 a ... 318.0 318.0 318.0 1874917 616.268760 104.0 20856831.0 1026.0 3347.0 1874917
PostgreSQL-32-8-81920-1-1 PostgreSQL-32-8-81920-1 PostgreSQL-32-8-81920 1 1 9hk6b 8 4 10240 30 a ... 302.0 302.0 302.0 1876646 621.082813 104.0 20922367.0 954.0 3241.0 1876646
PostgreSQL-32-8-81920-1-4 PostgreSQL-32-8-81920-1 PostgreSQL-32-8-81920 1 1 kfjzh 8 4 10240 30 a ... 683.0 683.0 683.0 1874551 615.696021 102.0 20201471.0 1009.0 3347.0 1874551
PostgreSQL-32-8-81920-1-5 PostgreSQL-32-8-81920-1 PostgreSQL-32-8-81920 1 1 krdkb 8 4 10240 30 a ... 412.0 412.0 412.0 1874516 578.652579 107.0 20545535.0 1008.0 3395.0 1874516
PostgreSQL-32-8-98304-1-1 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304 1 1 45zhr 8 4 12288 30 a ... 358.0 358.0 358.0 1874362 610.307512 105.0 18710527.0 1032.0 3517.0 1874362
PostgreSQL-32-8-98304-1-2 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304 1 1 bhvd9 8 4 12288 30 a ... 352.0 352.0 352.0 1874785 620.062738 98.0 18956287.0 1086.0 3495.0 1874785
PostgreSQL-32-8-98304-1-3 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304 1 1 k4r5r 8 4 12288 30 a ... 304.0 304.0 304.0 1873140 592.218152 101.0 18956287.0 1072.0 3579.0 1873140
PostgreSQL-32-8-98304-1-4 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304 1 1 kq9kf 8 4 12288 30 a ... 270.0 270.0 270.0 1876173 606.348524 114.0 18317311.0 1011.0 3465.0 1876173
PostgreSQL-32-8-98304-1-5 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304 1 1 mlt4b 8 4 12288 30 a ... 1324.0 1324.0 1324.0 1873546 621.741993 99.0 17874943.0 1015.0 3449.0 1873546
PostgreSQL-32-8-98304-1-6 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304 1 1 rksk7 8 4 12288 30 a ... 355.0 355.0 355.0 1876248 604.160926 107.0 18776063.0 966.0 3267.0 1876248
PostgreSQL-32-8-98304-1-8 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304 1 1 wjjww 8 4 12288 30 a ... 299.0 299.0 299.0 1874342 640.696887 104.0 18579455.0 1043.0 3525.0 1874342
PostgreSQL-32-8-98304-1-7 PostgreSQL-32-8-98304-1 PostgreSQL-32-8-98304 1 1 vn5b4 8 4 12288 30 a ... 991.0 991.0 991.0 1875485 621.688254 102.0 18759679.0 1080.0 3613.0 1875485
PostgreSQL-32-8-114688-1-4 PostgreSQL-32-8-114688-1 PostgreSQL-32-8-114688 1 1 hg76g 8 4 14336 30 a ... 282.0 282.0 282.0 1873433 598.484479 105.0 20119551.0 1020.0 3705.0 1873433
PostgreSQL-32-8-114688-1-1 PostgreSQL-32-8-114688-1 PostgreSQL-32-8-114688 1 1 2jq82 8 4 14336 30 a ... 420.0 420.0 420.0 1874504 569.857987 99.0 20135935.0 1010.0 3501.0 1874504
PostgreSQL-32-8-114688-1-2 PostgreSQL-32-8-114688-1 PostgreSQL-32-8-114688 1 1 8tw89 8 4 14336 30 a ... 352.0 352.0 352.0 1877114 599.267765 96.0 18825215.0 1047.0 3589.0 1877114
PostgreSQL-32-8-114688-1-3 PostgreSQL-32-8-114688-1 PostgreSQL-32-8-114688 1 1 fzvfv 8 4 14336 30 a ... 324.0 324.0 324.0 1875507 583.797519 105.0 13819903.0 1076.0 3627.0 1875507
PostgreSQL-32-8-114688-1-5 PostgreSQL-32-8-114688-1 PostgreSQL-32-8-114688 1 1 jwcxm 8 4 14336 30 a ... 335.0 335.0 335.0 1874515 588.496871 108.0 13819903.0 1063.0 3871.0 1874515
PostgreSQL-32-8-114688-1-6 PostgreSQL-32-8-114688-1 PostgreSQL-32-8-114688 1 1 knc8q 8 4 14336 30 a ... 374.0 374.0 374.0 1875905 634.300144 98.0 20103167.0 997.0 3533.0 1875905
PostgreSQL-32-8-114688-1-8 PostgreSQL-32-8-114688-1 PostgreSQL-32-8-114688 1 1 sf8fh 8 4 14336 30 a ... 301.0 301.0 301.0 1874330 616.061852 112.0 20299775.0 1029.0 3697.0 1874330
PostgreSQL-32-8-114688-1-7 PostgreSQL-32-8-114688-1 PostgreSQL-32-8-114688 1 1 kp9r9 8 4 14336 30 a ... 408.0 408.0 408.0 1875886 588.626170 105.0 18530303.0 999.0 3651.0 1875886
PostgreSQL-32-1-16384-1-1 PostgreSQL-32-1-16384-1 PostgreSQL-32-1-16384 1 1 dpz6r 1 32 16384 30 a ... 269.0 170.0 269.0 14999798 260.757995 102.0 67071.0 434.0 573.0 14999798
PostgreSQL-32-8-131072-1-8 PostgreSQL-32-8-131072-1 PostgreSQL-32-8-131072 1 1 qkjzm 8 4 16384 30 a ... 310.0 310.0 310.0 1875211 554.659974 108.0 18563071.0 953.0 3395.0 1875211
PostgreSQL-32-8-131072-1-2 PostgreSQL-32-8-131072-1 PostgreSQL-32-8-131072 1 1 82tzj 8 4 16384 30 a ... 358.0 358.0 358.0 1874712 614.491262 97.0 18939903.0 976.0 3417.0 1874712
PostgreSQL-32-8-131072-1-3 PostgreSQL-32-8-131072-1 PostgreSQL-32-8-131072 1 1 cdllj 8 4 16384 30 a ... 311.0 311.0 311.0 1875603 584.949657 111.0 18399231.0 909.0 3303.0 1875603
PostgreSQL-32-8-131072-1-4 PostgreSQL-32-8-131072-1 PostgreSQL-32-8-131072 1 1 dxbqh 8 4 16384 30 a ... 340.0 340.0 340.0 1873868 602.965952 106.0 20185087.0 912.0 3287.0 1873868
PostgreSQL-32-8-131072-1-5 PostgreSQL-32-8-131072-1 PostgreSQL-32-8-131072 1 1 fg7g6 8 4 16384 30 a ... 376.0 376.0 376.0 1876344 587.999810 103.0 20250623.0 923.0 3303.0 1876344
PostgreSQL-32-8-131072-1-6 PostgreSQL-32-8-131072-1 PostgreSQL-32-8-131072 1 1 g2lp7 8 4 16384 30 a ... 342.0 342.0 342.0 1875315 599.226824 96.0 18481151.0 987.0 3435.0 1875315
PostgreSQL-32-8-131072-1-7 PostgreSQL-32-8-131072-1 PostgreSQL-32-8-131072 1 1 gtfsx 8 4 16384 30 a ... 310.0 310.0 310.0 1875976 605.955583 93.0 13787135.0 948.0 3279.0 1875976
PostgreSQL-32-8-131072-1-1 PostgreSQL-32-8-131072-1 PostgreSQL-32-8-131072 1 1 6zf7s 8 4 16384 30 a ... 536.0 536.0 536.0 1875599 556.295206 105.0 13787135.0 961.0 3445.0 1875599
PostgreSQL-32-1-32768-1-1 PostgreSQL-32-1-32768-1 PostgreSQL-32-1-32768 1 1 k6vs7 1 32 32768 30 a ... 651.0 343.0 651.0 15000722 262.927871 101.0 722943.0 415.0 580.0 15000722
PostgreSQL-32-1-49152-1-1 PostgreSQL-32-1-49152-1 PostgreSQL-32-1-49152 1 1 klpvs 1 32 49152 30 a ... 2655.0 146.0 2655.0 14998304 454.032034 93.0 18235391.0 535.0 1687.0 14998304
PostgreSQL-32-1-65536-1-1 PostgreSQL-32-1-65536-1 PostgreSQL-32-1-65536 1 1 7w4z9 1 32 65536 30 a ... 343.0 168.0 343.0 14996210 644.124900 93.0 20774911.0 1075.0 3657.0 14996210
PostgreSQL-32-1-81920-1-1 PostgreSQL-32-1-81920-1 PostgreSQL-32-1-81920 1 1 chmc5 1 32 81920 30 a ... 938.0 175.0 938.0 15001854 630.408114 93.0 20758527.0 1115.0 3731.0 15001854
PostgreSQL-32-1-98304-1-1 PostgreSQL-32-1-98304-1 PostgreSQL-32-1-98304 1 1 7lqx2 1 32 98304 30 a ... 377.0 187.0 377.0 14995937 620.081984 93.0 18956287.0 1065.0 3641.0 14995937
PostgreSQL-32-1-114688-1-1 PostgreSQL-32-1-114688-1 PostgreSQL-32-1-114688 1 1 mwnxg 1 32 114688 30 a ... 305.0 178.0 305.0 15000966 635.884341 95.0 20987903.0 1041.0 3383.0 15000966
PostgreSQL-32-1-131072-1-1 PostgreSQL-32-1-131072-1 PostgreSQL-32-1-131072 1 1 zp9vj 1 32 131072 30 a ... 305.0 132.0 305.0 14999563 647.170531 94.0 20889599.0 1061.0 3619.0 14999563

72 rows × 43 columns

Plot Results per Number of Pods¶

In [35]:
%matplotlib inline

column = "threads"
x = "target"
y = "[OVERALL].Throughput(ops/sec)"
evaluation.plot(df_plot, column=column, x=x, y=y, plot_by="pod_count")
No description has been provided for this image
In [36]:
column = "threads"
x = "target"
y = "[READ].95thPercentileLatency(us)"
if y in df_plot.columns:
    evaluation.plot(df_plot, column=column, x=x, y=y, plot_by="experiment_run")

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
No description has been provided for this image

Aggregate by Parallel Pods¶

In [37]:
df = evaluation.get_df_benchmarking()
df.fillna(0, inplace=True)
#df.drop(df[df['[READ].Operations'] == 0].index, inplace=True)
#df.drop(df[df['[READ].Return=OK'] == 0].index, inplace=True)
df_plot = evaluation.benchmarking_set_datatypes(df)
df_aggregated = evaluation.benchmarking_aggregate_by_parallel_pods(df_plot)
df_aggregated.sort_values('target', inplace=True)
df_aggregated.T
Out[37]:
PostgreSQL-32-1-16384-1 PostgreSQL-32-8-16384-1 PostgreSQL-32-1-32768-1 PostgreSQL-32-8-32768-1 PostgreSQL-32-1-49152-1 PostgreSQL-32-8-49152-1 PostgreSQL-32-1-65536-1 PostgreSQL-32-8-65536-1 PostgreSQL-32-1-81920-1 PostgreSQL-32-8-81920-1 PostgreSQL-32-1-98304-1 PostgreSQL-32-8-98304-1 PostgreSQL-32-1-114688-1 PostgreSQL-32-8-114688-1 PostgreSQL-32-1-131072-1 PostgreSQL-32-8-131072-1
connection PostgreSQL-32-1-16384-1 PostgreSQL-32-8-16384-1 PostgreSQL-32-1-32768-1 PostgreSQL-32-8-32768-1 PostgreSQL-32-1-49152-1 PostgreSQL-32-8-49152-1 PostgreSQL-32-1-65536-1 PostgreSQL-32-8-65536-1 PostgreSQL-32-1-81920-1 PostgreSQL-32-8-81920-1 PostgreSQL-32-1-98304-1 PostgreSQL-32-8-98304-1 PostgreSQL-32-1-114688-1 PostgreSQL-32-8-114688-1 PostgreSQL-32-1-131072-1 PostgreSQL-32-8-131072-1
configuration PostgreSQL-32-1-16384 PostgreSQL-32-8-16384 PostgreSQL-32-1-32768 PostgreSQL-32-8-32768 PostgreSQL-32-1-49152 PostgreSQL-32-8-49152 PostgreSQL-32-1-65536 PostgreSQL-32-8-65536 PostgreSQL-32-1-81920 PostgreSQL-32-8-81920 PostgreSQL-32-1-98304 PostgreSQL-32-8-98304 PostgreSQL-32-1-114688 PostgreSQL-32-8-114688 PostgreSQL-32-1-131072 PostgreSQL-32-8-131072
experiment_run 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
client 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
pod dpz6r 28mm26qp9s77mw4cpwg6cxgj6jpmmprt4r9tfbqm k6vs7 2pf8r5szs7cfgn7j2j7flc8b4mt7d6p9tmsrt62c klpvs 5wh9k998gfghk57rvqs4sfzq5sgxpftqqdcxwgtc 7w4z9 5spjql25nllrgp7nws4xrh8zlvc88wwp69bz58tr chmc5 9hk6b9wm6bb7bh7kfjzhkrdkbtt4fmvhthfxn2w5 7lqx2 45zhrbhvd9k4r5rkq9kfmlt4brksk7vn5b4wjjww mwnxg 2jq828tw89fzvfvhg76gjwcxmknc8qkp9r9sf8fh zp9vj 6zf7s82tzjcdlljdxbqhfg7g6g2lp7gtfsxqkjzm
pod_count 1 8 1 8 1 8 1 8 1 8 1 8 1 8 1 8
threads 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32
target 16384 16384 32768 32768 49152 49152 65536 65536 81920 81920 98304 98304 114688 114688 131072 131072
sf 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30
workload a a a a a a a a a a a a a a a a
operations 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000 30000000
batchsize -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0 -1.0
[OVERALL].RunTime(ms) 1831314.0 1832430.0 915761.0 916820.0 657683.0 636295.0 523264.0 525847.0 521007.0 532786.0 514630.0 525430.0 515631.0 522245.0 521364.0 513763.0
[OVERALL].Throughput(ops/sec) 16381.68004 16377.623988 32759.639251 32738.135294 45614.680629 47358.596378 57332.436399 57515.975815 57580.800258 58539.257066 58294.308532 57996.79243 58181.141165 58176.917093 57541.372247 59503.634165
[CLEANUP].Operations 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32
[CLEANUP].AverageLatency(us) 68.09375 110.03125 100.96875 127.28125 174.59375 196.09375 122.875 170.875 138.46875 170.65625 126.4375 211.28125 104.75 164.15625 97.0625 164.84375
[CLEANUP].MinLatency(us) 35.0 46.0 36.0 49.0 55.0 61.0 70.0 62.0 64.0 65.0 64.0 61.0 60.0 65.0 62.0 64.0
[CLEANUP].MaxLatency(us) 269.0 292.0 651.0 344.0 2655.0 1652.0 343.0 608.0 938.0 683.0 377.0 1324.0 305.0 420.0 305.0 536.0
[CLEANUP].95thPercentileLatency(us) 170.0 292.0 343.0 344.0 146.0 1652.0 168.0 608.0 175.0 683.0 187.0 1324.0 178.0 420.0 132.0 536.0
[CLEANUP].99thPercentileLatency(us) 269.0 292.0 651.0 344.0 2655.0 1652.0 343.0 608.0 938.0 683.0 377.0 1324.0 305.0 420.0 305.0 536.0
[READ].Operations 15000202 15000829 14999278 14998494 15001696 15000538 15003790 15001673 14998146 14998594 15004063 15001919 14999034 14998806 15000437 14997372
[READ].AverageLatency(us) 236.30964 253.423811 244.156256 282.138179 349.143032 400.705718 419.172145 450.779241 432.829566 442.564595 424.969676 443.003051 414.153399 462.604994 418.644577 447.964995
[READ].MinLatency(us) 93.0 95.0 91.0 93.0 89.0 91.0 89.0 89.0 85.0 91.0 86.0 91.0 86.0 91.0 86.0 90.0
[READ].MaxLatency(us) 70271.0 172287.0 1073151.0 196095.0 18399231.0 21381119.0 18235391.0 21004287.0 20709375.0 20627455.0 18776063.0 18415615.0 20905983.0 20168703.0 20856831.0 20316159.0
[READ].95thPercentileLatency(us) 392.0 467.0 395.0 469.0 482.0 603.0 774.0 831.0 788.0 921.0 757.0 861.0 762.0 864.0 750.0 790.0
[READ].99thPercentileLatency(us) 514.0 638.0 540.0 908.0 1209.0 1850.0 2289.0 3101.0 2355.0 2325.0 2301.0 2543.0 2147.0 2779.0 2287.0 2471.0
[READ].Return=OK 15000202 15000829 14999278 14998494 15001696 15000538 15003790 15001673 14998146 14998594 15004063 15001919 14999034 14998806 15000437 14997372
[UPDATE].Operations 14999798 14999171 15000722 15001506 14998304 14999462 14996210 14998327 15001854 15001406 14995937 14998081 15000966 15001194 14999563 15002628
[UPDATE].AverageLatency(us) 260.757995 278.178307 262.927871 302.439357 454.032034 478.609792 644.1249 618.497339 630.408114 606.642677 620.081984 614.653123 635.884341 597.361598 647.170531 588.318033
[UPDATE].MinLatency(us) 102.0 108.0 101.0 103.0 93.0 101.0 93.0 100.0 93.0 98.0 93.0 98.0 95.0 96.0 94.0 93.0
[UPDATE].MaxLatency(us) 67071.0 130815.0 722943.0 1277951.0 18235391.0 21233663.0 20774911.0 21069823.0 20758527.0 20955135.0 18956287.0 18956287.0 20987903.0 20299775.0 20889599.0 20250623.0
[UPDATE].95thPercentileLatency(us) 434.0 515.0 415.0 508.0 535.0 675.0 1075.0 1096.0 1115.0 1088.0 1065.0 1086.0 1041.0 1076.0 1061.0 987.0
[UPDATE].99thPercentileLatency(us) 573.0 692.0 580.0 1008.0 1687.0 2353.0 3657.0 3931.0 3731.0 3395.0 3641.0 3613.0 3383.0 3871.0 3619.0 3445.0
[UPDATE].Return=OK 14999798 14999171 15000722 15001506 14998304 14999462 14996210 14998327 15001854 15001406 14995937 14998081 15000966 15001194 14999563 15002628

Group by Connection¶

In [38]:
df_plot = evaluation.benchmarking_set_datatypes(df)
df_groups = df_plot.groupby(['connection', 'pod'])

df_groups = df_groups.sum('target')
df_groups
Out[38]:
experiment_run client pod_count threads target sf operations batchsize [OVERALL].RunTime(ms) [OVERALL].Throughput(ops/sec) ... [CLEANUP].MaxLatency(us) [CLEANUP].95thPercentileLatency(us) [CLEANUP].99thPercentileLatency(us) [UPDATE].Operations [UPDATE].AverageLatency(us) [UPDATE].MinLatency(us) [UPDATE].MaxLatency(us) [UPDATE].95thPercentileLatency(us) [UPDATE].99thPercentileLatency(us) [UPDATE].Return=OK
connection pod
PostgreSQL-32-1-114688-1 mwnxg 1 1 1 32 114688 30 30000000 -1 515631.0 58181.141165 ... 305.0 178.0 305.0 15000966 635.884341 95.0 20987903.0 1041.0 3383.0 15000966
PostgreSQL-32-1-131072-1 zp9vj 1 1 1 32 131072 30 30000000 -1 521364.0 57541.372247 ... 305.0 132.0 305.0 14999563 647.170531 94.0 20889599.0 1061.0 3619.0 14999563
PostgreSQL-32-1-16384-1 dpz6r 1 1 1 32 16384 30 30000000 -1 1831314.0 16381.680040 ... 269.0 170.0 269.0 14999798 260.757995 102.0 67071.0 434.0 573.0 14999798
PostgreSQL-32-1-32768-1 k6vs7 1 1 1 32 32768 30 30000000 -1 915761.0 32759.639251 ... 651.0 343.0 651.0 15000722 262.927871 101.0 722943.0 415.0 580.0 15000722
PostgreSQL-32-1-49152-1 klpvs 1 1 1 32 49152 30 30000000 -1 657683.0 45614.680629 ... 2655.0 146.0 2655.0 14998304 454.032034 93.0 18235391.0 535.0 1687.0 14998304
PostgreSQL-32-1-65536-1 7w4z9 1 1 1 32 65536 30 30000000 -1 523264.0 57332.436399 ... 343.0 168.0 343.0 14996210 644.124900 93.0 20774911.0 1075.0 3657.0 14996210
PostgreSQL-32-1-81920-1 chmc5 1 1 1 32 81920 30 30000000 -1 521007.0 57580.800258 ... 938.0 175.0 938.0 15001854 630.408114 93.0 20758527.0 1115.0 3731.0 15001854
PostgreSQL-32-1-98304-1 7lqx2 1 1 1 32 98304 30 30000000 -1 514630.0 58294.308532 ... 377.0 187.0 377.0 14995937 620.081984 93.0 18956287.0 1065.0 3641.0 14995937
PostgreSQL-32-8-114688-1 2jq82 1 1 8 4 14336 30 3750000 -1 520977.0 7198.014500 ... 420.0 420.0 420.0 1874504 569.857987 99.0 20135935.0 1010.0 3501.0 1874504
8tw89 1 1 8 4 14336 30 3750000 -1 522245.0 7180.537870 ... 352.0 352.0 352.0 1877114 599.267765 96.0 18825215.0 1047.0 3589.0 1877114
fzvfv 1 1 8 4 14336 30 3750000 -1 518308.0 7235.080300 ... 324.0 324.0 324.0 1875507 583.797519 105.0 13819903.0 1076.0 3627.0 1875507
hg76g 1 1 8 4 14336 30 3750000 -1 499907.0 7501.395260 ... 282.0 282.0 282.0 1873433 598.484479 105.0 20119551.0 1020.0 3705.0 1873433
jwcxm 1 1 8 4 14336 30 3750000 -1 514709.0 7285.670155 ... 335.0 335.0 335.0 1874515 588.496871 108.0 13819903.0 1063.0 3871.0 1874515
knc8q 1 1 8 4 14336 30 3750000 -1 518773.0 7228.595166 ... 374.0 374.0 374.0 1875905 634.300144 98.0 20103167.0 997.0 3533.0 1875905
kp9r9 1 1 8 4 14336 30 3750000 -1 515101.0 7280.125645 ... 408.0 408.0 408.0 1875886 588.626170 105.0 18530303.0 999.0 3651.0 1875886
sf8fh 1 1 8 4 14336 30 3750000 -1 515996.0 7267.498198 ... 301.0 301.0 301.0 1874330 616.061852 112.0 20299775.0 1029.0 3697.0 1874330
PostgreSQL-32-8-131072-1 6zf7s 1 1 8 4 16384 30 3750000 -1 487554.0 7691.455716 ... 536.0 536.0 536.0 1875599 556.295206 105.0 13787135.0 961.0 3445.0 1875599
82tzj 1 1 8 4 16384 30 3750000 -1 512945.0 7310.725321 ... 358.0 358.0 358.0 1874712 614.491262 97.0 18939903.0 976.0 3417.0 1874712
cdllj 1 1 8 4 16384 30 3750000 -1 502692.0 7459.836242 ... 311.0 311.0 311.0 1875603 584.949657 111.0 18399231.0 909.0 3303.0 1875603
dxbqh 1 1 8 4 16384 30 3750000 -1 510550.0 7345.020076 ... 340.0 340.0 340.0 1873868 602.965952 106.0 20185087.0 912.0 3287.0 1873868
fg7g6 1 1 8 4 16384 30 3750000 -1 508747.0 7371.050837 ... 376.0 376.0 376.0 1876344 587.999810 103.0 20250623.0 923.0 3303.0 1876344
g2lp7 1 1 8 4 16384 30 3750000 -1 511584.0 7330.174517 ... 342.0 342.0 342.0 1875315 599.226824 96.0 18481151.0 987.0 3435.0 1875315
gtfsx 1 1 8 4 16384 30 3750000 -1 513763.0 7299.085376 ... 310.0 310.0 310.0 1875976 605.955583 93.0 13787135.0 948.0 3279.0 1875976
qkjzm 1 1 8 4 16384 30 3750000 -1 487248.0 7696.286080 ... 310.0 310.0 310.0 1875211 554.659974 108.0 18563071.0 953.0 3395.0 1875211
PostgreSQL-32-8-16384-1 28mm2 1 1 8 4 2048 30 3750000 -1 1831372.0 2047.645153 ... 238.0 238.0 238.0 1873921 259.960231 108.0 88511.0 461.0 614.0 1873921
6qp9s 1 1 8 4 2048 30 3750000 -1 1831557.0 2047.438327 ... 267.0 267.0 267.0 1875349 279.281535 111.0 130815.0 430.0 692.0 1875349
77mw4 1 1 8 4 2048 30 3750000 -1 1832430.0 2046.462894 ... 228.0 228.0 228.0 1874880 279.843256 109.0 77247.0 503.0 648.0 1874880
cpwg6 1 1 8 4 2048 30 3750000 -1 1831259.0 2047.771506 ... 244.0 244.0 244.0 1875427 269.904560 109.0 130175.0 415.0 649.0 1875427
cxgj6 1 1 8 4 2048 30 3750000 -1 1831962.0 2046.985691 ... 204.0 204.0 204.0 1875337 297.950520 109.0 76415.0 467.0 640.0 1875337
jpmmp 1 1 8 4 2048 30 3750000 -1 1831550.0 2047.446152 ... 292.0 292.0 292.0 1875014 261.791274 108.0 55775.0 398.0 625.0 1875014
rt4r9 1 1 8 4 2048 30 3750000 -1 1832282.0 2046.628194 ... 230.0 230.0 230.0 1873638 291.152009 114.0 118847.0 515.0 655.0 1873638
tfbqm 1 1 8 4 2048 30 3750000 -1 1831729.0 2047.246072 ... 215.0 215.0 215.0 1875605 285.543071 113.0 127295.0 491.0 633.0 1875605
PostgreSQL-32-8-32768-1 2pf8r 1 1 8 4 4096 30 3750000 -1 916321.0 4092.452317 ... 282.0 282.0 282.0 1874112 290.050854 109.0 97407.0 454.0 946.0 1874112
5szs7 1 1 8 4 4096 30 3750000 -1 916820.0 4090.224908 ... 236.0 236.0 236.0 1875046 295.300415 110.0 332287.0 437.0 1008.0 1875046
cfgn7 1 1 8 4 4096 30 3750000 -1 916751.0 4090.532762 ... 260.0 260.0 260.0 1874994 304.605591 103.0 1277951.0 471.0 971.0 1874994
j2j7f 1 1 8 4 4096 30 3750000 -1 916269.0 4092.684572 ... 324.0 324.0 324.0 1876703 312.227578 107.0 735231.0 478.0 929.0 1876703
lc8b4 1 1 8 4 4096 30 3750000 -1 916220.0 4092.903451 ... 264.0 264.0 264.0 1873702 282.308903 105.0 159871.0 430.0 929.0 1873702
mt7d6 1 1 8 4 4096 30 3750000 -1 916340.0 4092.367462 ... 323.0 323.0 323.0 1874082 309.469422 110.0 86015.0 508.0 823.0 1874082
p9tms 1 1 8 4 4096 30 3750000 -1 916454.0 4091.858402 ... 301.0 301.0 301.0 1875937 312.520923 109.0 539135.0 485.0 975.0 1875937
rt62c 1 1 8 4 4096 30 3750000 -1 915726.0 4095.111420 ... 344.0 344.0 344.0 1876930 313.031167 111.0 270079.0 482.0 971.0 1876930
PostgreSQL-32-8-49152-1 5wh9k 1 1 8 4 6144 30 3750000 -1 620373.0 6044.750497 ... 349.0 349.0 349.0 1875039 483.460633 105.0 12902399.0 625.0 2353.0 1875039
998gf 1 1 8 4 6144 30 3750000 -1 634777.0 5907.586444 ... 278.0 278.0 278.0 1874709 473.292028 101.0 18120703.0 637.0 2247.0 1874709
ghk57 1 1 8 4 6144 30 3750000 -1 635764.0 5898.415135 ... 425.0 425.0 425.0 1873652 469.840329 105.0 16007167.0 651.0 2293.0 1873652
rvqs4 1 1 8 4 6144 30 3750000 -1 636295.0 5893.492798 ... 1652.0 1652.0 1652.0 1875591 507.224280 105.0 18219007.0 658.0 2231.0 1875591
sfzq5 1 1 8 4 6144 30 3750000 -1 635374.0 5902.035651 ... 304.0 304.0 304.0 1874898 494.085958 107.0 21233663.0 654.0 2133.0 1874898
sgxpf 1 1 8 4 6144 30 3750000 -1 634520.0 5909.979197 ... 311.0 311.0 311.0 1874523 464.176237 107.0 18382847.0 641.0 2301.0 1874523
tqqdc 1 1 8 4 6144 30 3750000 -1 635739.0 5898.647086 ... 307.0 307.0 307.0 1874986 494.877304 101.0 14909439.0 675.0 2257.0 1874986
xwgtc 1 1 8 4 6144 30 3750000 -1 635196.0 5903.689570 ... 348.0 348.0 348.0 1876064 441.921570 105.0 14909439.0 646.0 2269.0 1876064
PostgreSQL-32-8-65536-1 5spjq 1 1 8 4 8192 30 3750000 -1 519071.0 7224.445211 ... 313.0 313.0 313.0 1875000 634.823838 100.0 21069823.0 1046.0 3931.0 1875000
l25nl 1 1 8 4 8192 30 3750000 -1 523360.0 7165.239988 ... 301.0 301.0 301.0 1875272 620.948668 101.0 20856831.0 1096.0 3859.0 1875272
lrgp7 1 1 8 4 8192 30 3750000 -1 517747.0 7242.919804 ... 297.0 297.0 297.0 1875517 609.462749 103.0 20922367.0 1087.0 3897.0 1875517
nws4x 1 1 8 4 8192 30 3750000 -1 520616.0 7203.005670 ... 332.0 332.0 332.0 1875071 622.472328 104.0 16457727.0 1053.0 3851.0 1875071
rh8zl 1 1 8 4 8192 30 3750000 -1 525847.0 7131.351895 ... 515.0 515.0 515.0 1875343 610.272305 100.0 20627455.0 1018.0 3753.0 1875343
vc88w 1 1 8 4 8192 30 3750000 -1 523591.0 7162.078798 ... 608.0 608.0 608.0 1875539 628.395195 102.0 20987903.0 1081.0 3865.0 1875539
wp69b 1 1 8 4 8192 30 3750000 -1 522241.0 7180.592868 ... 322.0 322.0 322.0 1874271 618.117459 103.0 20905983.0 1085.0 3907.0 1874271
z58tr 1 1 8 4 8192 30 3750000 -1 520375.0 7206.341581 ... 290.0 290.0 290.0 1872314 603.486167 107.0 13221887.0 1076.0 3917.0 1872314
PostgreSQL-32-8-81920-1 9hk6b 1 1 8 4 10240 30 3750000 -1 512036.0 7323.703802 ... 302.0 302.0 302.0 1876646 621.082813 104.0 20922367.0 954.0 3241.0 1876646
9wm6b 1 1 8 4 10240 30 3750000 -1 519087.0 7224.222529 ... 318.0 318.0 318.0 1874917 616.268760 104.0 20856831.0 1026.0 3347.0 1874917
b7bh7 1 1 8 4 10240 30 3750000 -1 503879.0 7442.262924 ... 384.0 384.0 384.0 1873757 583.579925 105.0 20496383.0 951.0 3315.0 1873757
kfjzh 1 1 8 4 10240 30 3750000 -1 523228.0 7167.047635 ... 683.0 683.0 683.0 1874551 615.696021 102.0 20201471.0 1009.0 3347.0 1874551
krdkb 1 1 8 4 10240 30 3750000 -1 492041.0 7621.316110 ... 412.0 412.0 412.0 1874516 578.652579 107.0 20545535.0 1008.0 3395.0 1874516
tt4fm 1 1 8 4 10240 30 3750000 -1 503374.0 7449.729227 ... 362.0 362.0 362.0 1875823 597.406558 113.0 20463615.0 972.0 3363.0 1875823
vhthf 1 1 8 4 10240 30 3750000 -1 515641.0 7272.501605 ... 320.0 320.0 320.0 1875332 621.982541 105.0 20430847.0 1007.0 3367.0 1875332
xn2w5 1 1 8 4 10240 30 3750000 -1 532786.0 7038.473233 ... 335.0 335.0 335.0 1875864 618.472218 98.0 20955135.0 1088.0 3275.0 1875864
PostgreSQL-32-8-98304-1 45zhr 1 1 8 4 12288 30 3750000 -1 517972.0 7239.773578 ... 358.0 358.0 358.0 1874362 610.307512 105.0 18710527.0 1032.0 3517.0 1874362
bhvd9 1 1 8 4 12288 30 3750000 -1 525430.0 7137.011591 ... 352.0 352.0 352.0 1874785 620.062738 98.0 18956287.0 1086.0 3495.0 1874785
k4r5r 1 1 8 4 12288 30 3750000 -1 510878.0 7340.304339 ... 304.0 304.0 304.0 1873140 592.218152 101.0 18956287.0 1072.0 3579.0 1873140
kq9kf 1 1 8 4 12288 30 3750000 -1 509697.0 7357.312286 ... 270.0 270.0 270.0 1876173 606.348524 114.0 18317311.0 1011.0 3465.0 1876173
mlt4b 1 1 8 4 12288 30 3750000 -1 523738.0 7160.068584 ... 1324.0 1324.0 1324.0 1873546 621.741993 99.0 17874943.0 1015.0 3449.0 1873546
rksk7 1 1 8 4 12288 30 3750000 -1 513476.0 7303.165094 ... 355.0 355.0 355.0 1876248 604.160926 107.0 18776063.0 966.0 3267.0 1876248
vn5b4 1 1 8 4 12288 30 3750000 -1 523770.0 7159.631136 ... 991.0 991.0 991.0 1875485 621.688254 102.0 18759679.0 1080.0 3613.0 1875485
wjjww 1 1 8 4 12288 30 3750000 -1 513732.0 7299.525823 ... 299.0 299.0 299.0 1874342 640.696887 104.0 18579455.0 1043.0 3525.0 1874342

72 rows × 30 columns

Get Maximum Number of Parallel Pods¶

In [39]:
max_pod_count = df_groups['pod_count'].max()
max_pod_count
Out[39]:
8

Plot Variations¶

Throughput¶

In [40]:
by='target'
column='[OVERALL].Throughput(ops/sec)'
groups = df_groups.groupby('pod_count')
#print(groups)
#print(len(groups))
rows = 1#(len(groups)+1)//2
#print(rows, "rows")
row=0
col=0
fig, axes = plt.subplots(nrows=rows, ncols=2, squeeze=False, figsize=(6,4*rows))#, sharex=True
#print(axes)
for key1, grp in groups:#df3.groupby(col1):
    #print(len(axs))
    #for key2, grp2 in grp.groupby(column):
    #print(grp2)
    #labels = "{} {}, {} {}".format(key1, 'pod_count', key2, column)
    #print(row,col)
    ax = grp.boxplot(ax=axes[row,col], by='target', column='[OVERALL].Throughput(ops/sec)', figsize=(6,4), layout=(rows,2), rot=90)
    #ax.set_ylim(0, df[y].max())
    col = col + 1
    if col > 1:
        row = row + 1
        col = 0
plt.legend(loc='best')
plt.tight_layout()
plt.show()

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
No description has been provided for this image
In [41]:
y = "[OVERALL].Throughput(ops/sec)"

plot_variation(df_groups, max_pod_count, y)
#ax = df_groups[df_groups['pod_count']==max_pod_count].boxplot(by='target', column=y, rot=90, figsize=(3,3), grid=False)
#ax.get_figure().suptitle('')
No description has been provided for this image

UPDATE¶

In [42]:
y='[UPDATE].99thPercentileLatency(us)'
plot_variation(df_groups, max_pod_count, y)
No description has been provided for this image

READ¶

In [43]:
y='[READ].AverageLatency(us)'
plot_variation(df_groups, max_pod_count, y)
No description has been provided for this image

Plot Comparison¶

Throughput¶

In [44]:
column = "pods"
x = "target"
y = "[OVERALL].Throughput(ops/sec)"
title = "Throughput [ops/s]"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)
No description has been provided for this image

UPDATE¶

In [45]:
column = "pods"
x = "target"
y = "[UPDATE].95thPercentileLatency(us)"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)
No description has been provided for this image
In [46]:
column = "pods"
x = "target"
y = "[UPDATE].99thPercentileLatency(us)"
title = "99th Lat UPDATE [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)
No description has been provided for this image
In [47]:
column = "pods"
x = "target"
y = "[UPDATE].AverageLatency(us)"
title = "Avg Lat UPDATE [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)
No description has been provided for this image
In [48]:
column = "pods"
x = "target"
y = "[UPDATE].MaxLatency(us)"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)
No description has been provided for this image
In [49]:
#column = ["threads", "pods"]
column = "pods"
x = "target"
y = "[UPDATE].99thPercentileLatency(us)"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)
No description has been provided for this image

READ¶

In [50]:
column = "pods"
x = "target"
y = "[READ].99thPercentileLatency(us)"
title = "99th Lat READ [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)
No description has been provided for this image
In [51]:
#column = ["threads", "pod_count"]
column = "pods"
x = "target"
y = "[READ].AverageLatency(us)"
title = "Avg Lat READ [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)
No description has been provided for this image
In [52]:
column = "pods"
x = "target"
y = "[READ].95thPercentileLatency(us)"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)
No description has been provided for this image

INSERT¶

In [53]:
column = "pods"
x = "target"
y = "[INSERT].99thPercentileLatency(us)"
title = "99th Lat INSERT [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)

[INSERT].99thPercentileLatency(us) not avalable
In [54]:
#column = ["threads", "pod_count"]
column = "pods"
x = "target"
y = "[INSERT].AverageLatency(us)"
title = "Avg Lat INSERT [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)

[INSERT].AverageLatency(us) not avalable
In [55]:
column = "pods"
x = "target"
y = "[INSERT].95thPercentileLatency(us)"
title = "95th Lat INSERT [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)

[INSERT].95thPercentileLatency(us) not avalable

SCAN¶

In [56]:
column = "pods"
x = "target"
y = "[SCAN].99thPercentileLatency(us)"
title = "99th Lat SCAN [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)

[SCAN].99thPercentileLatency(us) not avalable
In [57]:
#column = ["threads", "pod_count"]
column = "pods"
x = "target"
y = "[SCAN].AverageLatency(us)"
title = "Avg Lat SCAN [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)

[SCAN].AverageLatency(us) not avalable
In [58]:
column = "pods"
x = "target"
y = "[SCAN].95thPercentileLatency(us)"
title = "95th Lat SCAN [$\mu$s]"
plot_comparison(df_aggregated, x, y, column, title, peak_benchmark)

[SCAN].95thPercentileLatency(us) not avalable

Show Infos about Connections¶

In [59]:
import ast

with open(path+"/"+code+"/connections.config",'r') as inf:
    connections = ast.literal_eval(inf.read())

print("found", len(connections), "connections")

found 16 connections
In [60]:
import json
#pretty_connections = json.dumps(connections, indent=2)

#print(pretty_connections)

Loading Time per Number of Loading Threads¶

In [61]:
connections_sorted = sorted(connections, key=lambda c: c['name']) 

for c in connections_sorted:
    print(c['name'], 
          c['timeLoad'], 
          '[s] for', 
          c['parameter']['connection_parameter']['loading_parameters']['YCSB_THREADCOUNT'], 
          'threads on',
          c['hostsystem']['node'])

PostgreSQL-32-1-114688-1 1068.2056978940964 [s] for 32 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-1-131072-1 1087.3417482078075 [s] for 32 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-1-16384-1 1834.0904397517443 [s] for 32 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-1-32768-1 1062.0822292342782 [s] for 32 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-1-49152-1 1065.0029773414135 [s] for 32 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-1-65536-1 1033.139736406505 [s] for 32 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-1-81920-1 1056.9990959092975 [s] for 32 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-1-98304-1 1077.2663244530559 [s] for 32 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-8-114688-1 1129.2390964254737 [s] for 4 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-8-131072-1 1094.283723473549 [s] for 4 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-8-16384-1 1835.2130344808102 [s] for 4 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-8-32768-1 1074.2522376477718 [s] for 4 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-8-49152-1 1054.188381895423 [s] for 4 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-8-65536-1 1095.2031724601984 [s] for 4 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-8-81920-1 1076.1312694624066 [s] for 4 threads on ip-192-168-92-229.eu-central-1.compute.internal
PostgreSQL-32-8-98304-1 1073.1441322341561 [s] for 4 threads on ip-192-168-92-229.eu-central-1.compute.internal

Get monitoring metrics¶

Loading¶

Transform Monitoring Results to DataFrame¶

This has to be done only once. Transformed Results are stored in result folder.

We also show a list of available metrics.

In [62]:
evaluation.transform_monitoring_results(component="loading")

evaluation.get_monitoring_metrics()
Out[62]:
['total_cpu_memory',
 'total_cpu_memory_cached',
 'total_cpu_util',
 'total_cpu_util_max',
 'total_cpu_throttled',
 'total_cpu_util_others',
 'total_cpu_util_s',
 'total_cpu_util_user_s',
 'total_cpu_util_sys_s',
 'total_cpu_throttled_s',
 'total_cpu_util_others_s',
 'total_network_rx',
 'total_network_tx',
 'total_fs_read',
 'total_fs_write',
 'total_gpu_util',
 'total_gpu_power',
 'total_gpu_memory']

Plot all Metrics¶

In [63]:
list_metrics = evaluation.get_monitoring_metrics()
row=0
col=0
rows = (len(list_metrics)+1)//2

fig, axes = plt.subplots(nrows=rows, ncols=2, sharex=True, squeeze=False, figsize=(12,8*rows))
for metric in list_metrics:
    df = evaluation.get_monitoring_metric(metric)
    ax = df.plot(ax=axes[row,col], kind='line', title=metric, layout=(rows,2))
    col = col + 1
    if col > 1:
        row = row + 1
        col = 0
plt.legend(loc='best')
plt.tight_layout()
plt.show()
No description has been provided for this image

Plot some Aggregations¶

Compute Time¶

In [64]:
df = evaluation.get_monitoring_metric('total_cpu_util_s', component='loading').max() - evaluation.get_monitoring_metric('total_cpu_util_s', component='loading').min()
plot_metric(df, "Compute Time")
No description has been provided for this image

Maximum CPU Util¶

In [65]:
df = evaluation.get_monitoring_metric('total_cpu_util', component='loading').max()
plot_metric(df, "Max CPU")
#df.plot.bar()
No description has been provided for this image

Maximum Util of a CPU Core¶

In [66]:
df = evaluation.get_monitoring_metric('total_cpu_util_max', component='loading').max()*100
plot_metric(df, "Max CPU Core [%]")
No description has been provided for this image

Maxmimum RAM¶

In [67]:
df = evaluation.get_monitoring_metric('total_cpu_memory', component='loading').max()/1000
plot_metric(df, "Max RAM [Gb]")
No description has been provided for this image

Maxmimum Cache¶

In [68]:
df = evaluation.get_monitoring_metric('total_cpu_memory_cached', component='loading').max()/1000
plot_metric(df, "Max Cache [Gb]")
No description has been provided for this image

Loader Driver¶

Transform Monitoring Results to DataFrame¶

This has to be done only once. Transformed Results are stored in result folder.

We also show a list of available metrics.

In [69]:
evaluation.transform_monitoring_results(component='loader')

evaluation.get_monitoring_metrics()
Out[69]:
['total_cpu_memory',
 'total_cpu_memory_cached',
 'total_cpu_util',
 'total_cpu_util_max',
 'total_cpu_throttled',
 'total_cpu_util_others',
 'total_cpu_util_s',
 'total_cpu_util_user_s',
 'total_cpu_util_sys_s',
 'total_cpu_throttled_s',
 'total_cpu_util_others_s',
 'total_network_rx',
 'total_network_tx',
 'total_fs_read',
 'total_fs_write',
 'total_gpu_util',
 'total_gpu_power',
 'total_gpu_memory']

Plot all Metrics¶

In [70]:
list_metrics = evaluation.get_monitoring_metrics()
row=0
col=0
rows = (len(list_metrics)+1)//2

fig, axes = plt.subplots(nrows=rows, ncols=2, sharex=True, squeeze=False, figsize=(12,8*rows))
for metric in list_metrics:
    df = evaluation.get_monitoring_metric(metric, component='loader')
    #if df.sum().sum() > 0:
    ax = df.plot(ax=axes[row,col], kind='line', title=metric, layout=(rows,2))
    col = col + 1
    if col > 1:
        row = row + 1
        col = 0
plt.legend(loc='best')
plt.tight_layout()
plt.show()
No description has been provided for this image

Plot some Aggregations¶

Compute Time¶

In [71]:
df = evaluation.get_monitoring_metric('total_cpu_util_s', component='loader').max() - evaluation.get_monitoring_metric('total_cpu_util_s', component='loader').min()
plot_metric(df, "Compute Time")
No description has been provided for this image

Maxmimum CPU Util¶

In [72]:
df = evaluation.get_monitoring_metric('total_cpu_util', component='loader').max()
plot_metric(df, "Max CPU")
No description has been provided for this image

Maximum Util of a CPU Core¶

In [73]:
df = evaluation.get_monitoring_metric('total_cpu_util_max', component='loader').max()*100
plot_metric(df, "Max CPU Core [%]")
No description has been provided for this image

Maximum RAM¶

In [74]:
df = evaluation.get_monitoring_metric('total_cpu_memory', component='loader').max()/1000
plot_metric(df, "Max RAM [Gb]")
No description has been provided for this image

Maximum Cache¶

In [75]:
df = evaluation.get_monitoring_metric('total_cpu_memory_cached', component='loader').max()/1000
plot_metric(df, "Max Cache")
No description has been provided for this image

Benchmarking¶

Transform Monitoring Results to DataFrame¶

This has to be done only once. Transformed Results are stored in result folder.

We also show a list of available metrics.

In [76]:
evaluation.transform_monitoring_results(component='stream')

evaluation.get_monitoring_metrics()
Out[76]:
['total_cpu_memory',
 'total_cpu_memory_cached',
 'total_cpu_util',
 'total_cpu_util_max',
 'total_cpu_throttled',
 'total_cpu_util_others',
 'total_cpu_util_s',
 'total_cpu_util_user_s',
 'total_cpu_util_sys_s',
 'total_cpu_throttled_s',
 'total_cpu_util_others_s',
 'total_network_rx',
 'total_network_tx',
 'total_fs_read',
 'total_fs_write',
 'total_gpu_util',
 'total_gpu_power',
 'total_gpu_memory']

Plot all Metrics¶

In [77]:
list_metrics = evaluation.get_monitoring_metrics()
row=0
col=0
rows = (len(list_metrics)+1)//2

fig, axes = plt.subplots(nrows=rows, ncols=2, sharex=True, squeeze=False, figsize=(12,8*rows))
for metric in list_metrics:
    df = evaluation.get_monitoring_metric(metric, component='stream')
    #if df.sum().sum() > 0:
    ax = df.plot(ax=axes[row,col], kind='line', title=metric, layout=(rows,2))
    col = col + 1
    if col > 1:
        row = row + 1
        col = 0
plt.legend(loc='best')
plt.tight_layout()
plt.show()
No description has been provided for this image

Plot some Aggregations¶

Compute Time¶

In [78]:
df = evaluation.get_monitoring_metric('total_cpu_util_s', component='stream').max() - evaluation.get_monitoring_metric('total_cpu_util_s', component='stream').min()
plot_metric(df, "SUT compute [CPUs]", peak_benchmark)
df
Out[78]:
PostgreSQL-32-1-16384-1     3331.054276
PostgreSQL-32-1-32768-1     2135.833365
PostgreSQL-32-1-49152-1     2860.510238
PostgreSQL-32-1-65536-1     4950.739413
PostgreSQL-32-1-81920-1     5049.445946
PostgreSQL-32-1-98304-1     4894.388010
PostgreSQL-32-1-114688-1    4695.229749
PostgreSQL-32-1-131072-1    4854.766119
PostgreSQL-32-8-16384-1     3250.851061
PostgreSQL-32-8-32768-1     2153.290309
PostgreSQL-32-8-49152-1     2969.004657
PostgreSQL-32-8-65536-1     4392.625075
PostgreSQL-32-8-81920-1     4174.933585
PostgreSQL-32-8-98304-1     4309.111984
PostgreSQL-32-8-114688-1    4379.602404
PostgreSQL-32-8-131072-1    4079.609299
dtype: float64
No description has been provided for this image

Maximum CPU Util¶

In [79]:
df = evaluation.get_monitoring_metric('total_cpu_util', component='stream').max()
plot_metric(df, "SUT Max CPU", peak_benchmark)
df
Out[79]:
PostgreSQL-32-1-16384-1      4.973462
PostgreSQL-32-1-32768-1      4.478603
PostgreSQL-32-1-49152-1     13.785709
PostgreSQL-32-1-65536-1     16.049334
PostgreSQL-32-1-81920-1     16.826042
PostgreSQL-32-1-98304-1     15.550890
PostgreSQL-32-1-114688-1    13.336056
PostgreSQL-32-1-131072-1    13.245598
PostgreSQL-32-8-16384-1      2.856713
PostgreSQL-32-8-32768-1      3.039626
PostgreSQL-32-8-49152-1     11.866286
PostgreSQL-32-8-65536-1     14.107406
PostgreSQL-32-8-81920-1     13.991398
PostgreSQL-32-8-98304-1     13.454298
PostgreSQL-32-8-114688-1    15.131495
PostgreSQL-32-8-131072-1    15.122374
dtype: float64
No description has been provided for this image

Maximum Util of a CPU Core¶

In [80]:
df = evaluation.get_monitoring_metric('total_cpu_util_max', component='stream').max()*100
plot_metric(df, "SUT Max CPU Core", peak_benchmark)
No description has been provided for this image

Maximum RAM¶

In [81]:
df = evaluation.get_monitoring_metric('total_cpu_memory', component='stream').max()
plot_metric(df, "SUT Max RAM", peak_benchmark)
No description has been provided for this image

Maximum Cache¶

In [82]:
df = evaluation.get_monitoring_metric('total_cpu_memory_cached', component='stream').max()
plot_metric(df, "SUT Max Cache", peak_benchmark)
No description has been provided for this image

Benchmarker Driver¶

Transform Monitoring Results to DataFrame¶

This has to be done only once. Transformed Results are stored in result folder.

We also show a list of available metrics.

In [83]:
evaluation.transform_monitoring_results(component='benchmarker')

evaluation.get_monitoring_metrics()
Out[83]:
['total_cpu_memory',
 'total_cpu_memory_cached',
 'total_cpu_util',
 'total_cpu_util_max',
 'total_cpu_throttled',
 'total_cpu_util_others',
 'total_cpu_util_s',
 'total_cpu_util_user_s',
 'total_cpu_util_sys_s',
 'total_cpu_throttled_s',
 'total_cpu_util_others_s',
 'total_network_rx',
 'total_network_tx',
 'total_fs_read',
 'total_fs_write',
 'total_gpu_util',
 'total_gpu_power',
 'total_gpu_memory']

Plot all Metrics¶

In [84]:
list_metrics = evaluation.get_monitoring_metrics()
row=0
col=0
rows = (len(list_metrics)+1)//2

fig, axes = plt.subplots(nrows=rows, ncols=2, sharex=True, squeeze=False, figsize=(12,8*rows))
for metric in list_metrics:
    df = evaluation.get_monitoring_metric(metric, component='benchmarker')
    #if df.sum().sum() > 0:
    ax = df.plot(ax=axes[row,col], kind='line', title=metric, layout=(rows,2))
    col = col + 1
    if col > 1:
        row = row + 1
        col = 0
plt.legend(loc='best')
plt.tight_layout()
plt.show()
No description has been provided for this image

Plot some Aggregations¶

Compute Time¶

In [85]:
df = evaluation.get_monitoring_metric('total_cpu_util_s', component='benchmarker').max() - evaluation.get_monitoring_metric('total_cpu_util_s', component='benchmarker').min()
plot_metric(df, "Compute Time", peak_benchmark)
No description has been provided for this image
In [86]:
df = evaluation.get_monitoring_metric('total_cpu_util_s', component='benchmarker').max() - evaluation.get_monitoring_metric('total_cpu_util_s', component='benchmarker').min()
plot_metric(df, "Driver compute [CPUs]", peak_benchmark)
No description has been provided for this image

Maximum CPU Util¶

In [87]:
df = evaluation.get_monitoring_metric('total_cpu_util', component='benchmarker').max()
plot_metric(df, "Driver max CPU", peak_benchmark)
No description has been provided for this image

Maximum Util of a CPU Core¶

In [88]:
df = evaluation.get_monitoring_metric('total_cpu_util_max', component='benchmarker').max()*100
plot_metric(df, "Driver core max [%]", peak_benchmark)
plt.legend(loc='lower right', title='num pods')
Out[88]:
<matplotlib.legend.Legend at 0x20b2a2fa100>
No description has been provided for this image

Maximum RAM¶

In [89]:
df = evaluation.get_monitoring_metric('total_cpu_memory', component='benchmarker').max()/1000
plot_metric(df, "Max RAM [Gb]", peak_benchmark)
No description has been provided for this image

Maximum Cache¶

In [90]:
df = evaluation.get_monitoring_metric('total_cpu_memory_cached', component='benchmarker').max()
plot_metric(df, "Max Cache [Gb]", peak_benchmark)
No description has been provided for this image
In [ ]: